00000800:00000200:3.1F:1775764444.350075:0:2423:0:(socklnd.c:214:ksocknal_find_peer_locked()) got peer_ni [00000000b0d7ad93] -> 12345-192.168.202.155@tcp (2) 00000800:00000200:3.1:1775764444.405641:0:2413:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 1, ksnr_connected 2 ksnr_max_conns 1 00000800:00000200:3.1:1775764444.405651:0:2413:0:(socklnd_cb.c:748:ksocknal_queue_tx_locked()) Sending to 12345-192.168.202.155@tcp ip 192.168.202.155:988 00000800:00000200:3.1:1775764444.405655:0:2413:0:(socklnd_cb.c:767:ksocknal_queue_tx_locked()) Packet 0000000063c88d12 type 2, nob 96 niov 1 nkiov 0 00000800:00000200:1.1F:1775764444.424172:0:2413:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 2, ksnr_connected 6 ksnr_max_conns 1 00000800:00000200:3.1:1775764444.481033:0:2413:0:(socklnd.c:384:ksocknal_incr_conn_count()) Add conn type 3, ksnr_connected e ksnr_max_conns 1 00000080:00200000:1.0:1775764895.533542:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a6a136a0 22459: 1< 0 - - > 2< 91992064 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8671232 - | + - + - > 5< + - + - | 0 - | 1979 - -> 00000080:00200000:1.0:1775764895.533545:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000dd735606 00000080:00200000:1.0:1775764895.533546:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.533548:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.533556:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.533563:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.533571:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.533579:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.533590:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.533593:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.533595:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.533606:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.533609:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.533611:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.533615:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.533624:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.533632:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.533634:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.533638:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.533640:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22459, 22459] 00000080:00200000:1.0:1775764895.533650:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.533654:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.533683:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 91996160 i_blocks 90112 00000080:00200000:1.0:1775764895.533687:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.533705:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 91996160, count: 4096 00000080:00200000:1.0:1775764895.533715:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 91996160, bytes: 4096 00000080:00200000:1.0:1775764895.533718:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.533722:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22461, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.533723:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [91996160, 92000256) 0 0 00000080:00200000:1.0:1775764895.533726:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.533731:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22460, 22460] 00020000:00200000:1.0:1775764895.533733:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 91996160 chunk: [91996160, 92000256) 92000256, 4096 00020000:00200000:1.0:1775764895.533736:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.533749:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22460, 22460] 00000008:00010000:1.0:1775764895.533754:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.533756:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.533764:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.533773:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (91996160 92000255) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.533781:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.533789:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.533800:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.533808:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [91996160, 92000256) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.533813:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22460 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.533832:0:18302:0:(rw26.c:858:ll_write_end()) pos 91996160, len 4096, copied 4096 00000080:00200000:1.0:1775764895.533836:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000019e07e78[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.533839:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000023d7fa96 17ffffc0001001 3:0 ffff90c20b3a69a0 22460 lru 00000080:00200000:1.0:1775764895.533845:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002d3c6f5d 22460: 1< 0 - - > 2< 91996160 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8667136 - | + - + - > 5< + - + - | 0 - | 1980 - -> 00000080:00200000:1.0:1775764895.533847:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000019e07e78 00000080:00200000:1.0:1775764895.533848:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.533850:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.533858:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.533865:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.533896:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.533903:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.533915:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.533919:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.533921:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.533931:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.533934:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.533936:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.533940:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.533948:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.533956:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.533958:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.533962:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.533965:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22460, 22460] 00000080:00200000:1.0:1775764895.533976:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.533979:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.534007:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92000256 i_blocks 90112 00000080:00200000:1.0:1775764895.534011:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.534026:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92000256, count: 4096 00000080:00200000:1.0:1775764895.534036:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92000256, bytes: 4096 00000080:00200000:1.0:1775764895.534039:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.534042:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22462, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.534044:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92000256, 92004352) 0 0 00000080:00200000:1.0:1775764895.534046:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.534051:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22461, 22461] 00020000:00200000:1.0:1775764895.534054:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92000256 chunk: [92000256, 92004352) 92004352, 4096 00020000:00200000:1.0:1775764895.534056:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.534067:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22461, 22461] 00000008:00010000:1.0:1775764895.534073:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.534075:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.534083:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534092:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92000256 92004351) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534100:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534107:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534114:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.534135:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92000256, 92004352) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.534140:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22461 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.534156:0:18302:0:(rw26.c:858:ll_write_end()) pos 92000256, len 4096, copied 4096 00000080:00200000:1.0:1775764895.534160:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e1c40cfb[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.534163:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000062847f77 17ffffc0001001 3:0 ffff90c20b3a62a0 22461 lru 00000080:00200000:1.0:1775764895.534169:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a5842c24 22461: 1< 0 - - > 2< 92000256 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8663040 - | + - + - > 5< + - + - | 0 - | 1981 - -> 00000080:00200000:1.0:1775764895.534171:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e1c40cfb 00000080:00200000:1.0:1775764895.534172:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.534174:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.534182:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.534189:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.534196:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.534202:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.534213:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.534216:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.534218:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.534228:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.534230:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.534233:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.534237:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534244:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.534252:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.534254:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.534258:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.534261:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22461, 22461] 00000080:00200000:1.0:1775764895.534270:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.534273:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.534298:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92004352 i_blocks 90112 00000080:00200000:1.0:1775764895.534301:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.534316:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92004352, count: 4096 00000080:00200000:1.0:1775764895.534325:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92004352, bytes: 4096 00000080:00200000:1.0:1775764895.534328:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.534331:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22463, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.534332:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92004352, 92008448) 0 0 00000080:00200000:1.0:1775764895.534334:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.534339:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22462, 22462] 00020000:00200000:1.0:1775764895.534342:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92004352 chunk: [92004352, 92008448) 92008448, 4096 00020000:00200000:1.0:1775764895.534344:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.534353:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22462, 22462] 00000008:00010000:1.0:1775764895.534358:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.534361:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.534368:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534376:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92004352 92008447) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534384:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534391:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534398:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.534405:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92004352, 92008448) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.534410:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22462 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.534426:0:18302:0:(rw26.c:858:ll_write_end()) pos 92004352, len 4096, copied 4096 00000080:00200000:1.0:1775764895.534430:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005c5299cc[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.534432:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000071931bf9 17ffffc0001001 3:0 ffff90c20b3a6d20 22462 lru 00000080:00200000:1.0:1775764895.534438:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000004bf0856 22462: 1< 0 - - > 2< 92004352 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8658944 - | + - + - > 5< + - + - | 0 - | 1982 - -> 00000080:00200000:1.0:1775764895.534439:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005c5299cc 00000080:00200000:1.0:1775764895.534440:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.534442:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.534449:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.534457:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.534463:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.534470:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.534480:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.534482:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.534485:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.534494:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.534496:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.534498:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.534502:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534509:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.534517:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.534519:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.534523:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.534526:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22462, 22462] 00000080:00200000:1.0:1775764895.534536:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.534540:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.534568:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92008448 i_blocks 90112 00000080:00200000:1.0:1775764895.534572:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.534588:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92008448, count: 4096 00000080:00200000:1.0:1775764895.534598:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92008448, bytes: 4096 00000080:00200000:1.0:1775764895.534601:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.534604:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22464, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.534606:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92008448, 92012544) 0 0 00000080:00200000:1.0:1775764895.534608:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.534613:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22463, 22463] 00020000:00200000:1.0:1775764895.534616:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92008448 chunk: [92008448, 92012544) 92012544, 4096 00020000:00200000:1.0:1775764895.534618:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.534631:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22463, 22463] 00000008:00010000:1.0:1775764895.534636:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.534638:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.534646:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534655:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92008448 92012543) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534662:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534675:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534682:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.534690:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92008448, 92012544) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.534696:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22463 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.534711:0:18302:0:(rw26.c:858:ll_write_end()) pos 92008448, len 4096, copied 4096 00000080:00200000:1.0:1775764895.534715:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000019eb0b82[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.534717:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000044436e00 17ffffc0001001 3:0 ffff90c20b3a6380 22463 lru 00000080:00200000:1.0:1775764895.534722:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000044eb1698 22463: 1< 0 - - > 2< 92008448 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8654848 - | + - + - > 5< + - + - | 0 - | 1983 - -> 00000080:00200000:1.0:1775764895.534724:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000019eb0b82 00000080:00200000:1.0:1775764895.534725:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.534726:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.534734:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.534741:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.534747:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.534753:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.534769:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.534772:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.534774:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.534784:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.534786:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.534789:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.534793:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534801:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.534808:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.534811:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.534815:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.534817:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22463, 22463] 00000080:00200000:1.0:1775764895.534828:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.534831:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.534860:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92012544 i_blocks 90112 00000080:00200000:1.0:1775764895.534864:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.534903:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92012544, count: 4096 00000080:00200000:1.0:1775764895.534915:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92012544, bytes: 4096 00000080:00200000:1.0:1775764895.534918:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.534921:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22465, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.534923:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92012544, 92016640) 0 0 00000080:00200000:1.0:1775764895.534925:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.534930:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22464, 22464] 00020000:00200000:1.0:1775764895.534933:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92012544 chunk: [92012544, 92016640) 92016640, 4096 00020000:00200000:1.0:1775764895.534936:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.534948:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22464, 22464] 00000008:00010000:1.0:1775764895.534952:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.534955:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.534963:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534972:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92012544 92016639) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534980:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534988:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.534994:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.535001:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92012544, 92016640) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.535006:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22464 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.535027:0:18302:0:(rw26.c:858:ll_write_end()) pos 92012544, len 4096, copied 4096 00000080:00200000:1.0:1775764895.535031:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000628a3a38[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.535034:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c4082371 17ffffc0001001 3:0 ffff90c20b3a6620 22464 lru 00000080:00200000:1.0:1775764895.535040:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002cb74eaa 22464: 1< 0 - - > 2< 92012544 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8650752 - | + - + - > 5< + - + - | 0 - | 1984 - -> 00000080:00200000:1.0:1775764895.535042:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000628a3a38 00000080:00200000:1.0:1775764895.535043:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.535045:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.535053:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535060:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535067:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535074:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.535085:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.535088:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.535090:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.535099:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.535102:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.535105:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.535108:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535116:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.535137:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.535140:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.535144:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.535147:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22464, 22464] 00000080:00200000:1.0:1775764895.535158:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.535161:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.535190:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92016640 i_blocks 90112 00000080:00200000:1.0:1775764895.535193:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.535209:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92016640, count: 4096 00000080:00200000:1.0:1775764895.535219:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92016640, bytes: 4096 00000080:00200000:1.0:1775764895.535222:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.535226:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22466, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.535227:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92016640, 92020736) 0 0 00000080:00200000:1.0:1775764895.535230:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.535235:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22465, 22465] 00020000:00200000:1.0:1775764895.535238:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92016640 chunk: [92016640, 92020736) 92020736, 4096 00020000:00200000:1.0:1775764895.535240:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.535251:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22465, 22465] 00000008:00010000:1.0:1775764895.535256:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.535258:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.535266:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535275:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92016640 92020735) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535282:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535289:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535295:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.535304:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92016640, 92020736) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.535309:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22465 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.535326:0:18302:0:(rw26.c:858:ll_write_end()) pos 92016640, len 4096, copied 4096 00000080:00200000:1.0:1775764895.535330:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004e01d908[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.535333:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b6c59388 17ffffc0001001 3:0 ffff90c20b3a6e00 22465 lru 00000080:00200000:1.0:1775764895.535339:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006251b63c 22465: 1< 0 - - > 2< 92016640 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8646656 - | + - + - > 5< + - + - | 0 - | 1985 - -> 00000080:00200000:1.0:1775764895.535341:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004e01d908 00000080:00200000:1.0:1775764895.535342:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.535344:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.535352:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535359:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535366:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535373:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.535384:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.535387:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.535389:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.535398:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.535401:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.535404:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.535408:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535415:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.535424:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.535426:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.535430:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.535432:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22465, 22465] 00000080:00200000:1.0:1775764895.535443:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.535446:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.535474:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92020736 i_blocks 90112 00000080:00200000:1.0:1775764895.535478:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.535494:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92020736, count: 4096 00000080:00200000:1.0:1775764895.535504:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92020736, bytes: 4096 00000080:00200000:1.0:1775764895.535507:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.535510:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22467, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.535512:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92020736, 92024832) 0 0 00000080:00200000:1.0:1775764895.535514:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.535518:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22466, 22466] 00020000:00200000:1.0:1775764895.535521:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92020736 chunk: [92020736, 92024832) 92024832, 4096 00020000:00200000:1.0:1775764895.535523:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.535533:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22466, 22466] 00000008:00010000:1.0:1775764895.535539:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.535541:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.535548:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535557:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92020736 92024831) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535565:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535572:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535580:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.535588:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92020736, 92024832) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.535593:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22466 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.535609:0:18302:0:(rw26.c:858:ll_write_end()) pos 92020736, len 4096, copied 4096 00000080:00200000:1.0:1775764895.535614:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c83c0c3d[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.535617:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004dd0429e 17ffffc0001001 3:0 ffff90c20b3a6540 22466 lru 00000080:00200000:1.0:1775764895.535622:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e92c193a 22466: 1< 0 - - > 2< 92020736 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8642560 - | + - + - > 5< + - + - | 0 - | 1986 - -> 00000080:00200000:1.0:1775764895.535625:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c83c0c3d 00000080:00200000:1.0:1775764895.535626:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.535628:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.535635:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535643:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535650:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535660:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.535671:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.535674:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.535676:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.535685:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.535688:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.535690:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.535694:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535702:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.535710:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.535712:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.535716:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.535718:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22466, 22466] 00000080:00200000:1.0:1775764895.535729:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.535732:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.535761:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92024832 i_blocks 90112 00000080:00200000:1.0:1775764895.535765:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.535780:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92024832, count: 4096 00000080:00200000:1.0:1775764895.535791:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92024832, bytes: 4096 00000080:00200000:1.0:1775764895.535794:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.535798:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22468, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.535799:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92024832, 92028928) 0 0 00000080:00200000:1.0:1775764895.535802:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.535807:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22467, 22467] 00020000:00200000:1.0:1775764895.535810:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92024832 chunk: [92024832, 92028928) 92028928, 4096 00020000:00200000:1.0:1775764895.535812:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.535825:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22467, 22467] 00000008:00010000:1.0:1775764895.535829:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.535832:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.535840:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535849:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92024832 92028927) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535858:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535865:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.535893:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.535902:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92024832, 92028928) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.535908:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22467 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.535935:0:18302:0:(rw26.c:858:ll_write_end()) pos 92024832, len 4096, copied 4096 00000080:00200000:1.0:1775764895.535940:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000097b7886c[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.535943:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e10eb499 17ffffc0001001 3:0 ffff90c20b3c8540 22467 lru 00000080:00200000:1.0:1775764895.535948:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000096e54b33 22467: 1< 0 - - > 2< 92024832 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8638464 - | + - + - > 5< + - + - | 0 - | 1987 - -> 00000080:00200000:1.0:1775764895.535950:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000097b7886c 00000080:00200000:1.0:1775764895.535951:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.535953:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.535961:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535969:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535976:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.535983:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.535995:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.535998:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.536000:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.536009:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.536012:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.536014:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.536018:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536026:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.536035:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.536038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.536042:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.536045:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22467, 22467] 00000080:00200000:1.0:1775764895.536056:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.536059:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.536088:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92028928 i_blocks 90112 00000080:00200000:1.0:1775764895.536092:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.536106:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92028928, count: 4096 00000080:00200000:1.0:1775764895.536115:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92028928, bytes: 4096 00000080:00200000:1.0:1775764895.536118:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.536134:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22469, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.536136:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92028928, 92033024) 0 0 00000080:00200000:1.0:1775764895.536138:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.536143:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22468, 22468] 00020000:00200000:1.0:1775764895.536146:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92028928 chunk: [92028928, 92033024) 92033024, 4096 00020000:00200000:1.0:1775764895.536149:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.536160:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22468, 22468] 00000008:00010000:1.0:1775764895.536165:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.536167:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.536175:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536183:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92028928 92033023) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536191:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536199:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536211:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.536218:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92028928, 92033024) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.536223:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22468 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.536239:0:18302:0:(rw26.c:858:ll_write_end()) pos 92028928, len 4096, copied 4096 00000080:00200000:1.0:1775764895.536243:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009f6146f8[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.536246:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000461167ea 17ffffc0001001 3:0 ffff90c20b3c8460 22468 lru 00000080:00200000:1.0:1775764895.536250:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000794c069c 22468: 1< 0 - - > 2< 92028928 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8634368 - | + - + - > 5< + - + - | 0 - | 1988 - -> 00000080:00200000:1.0:1775764895.536252:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009f6146f8 00000080:00200000:1.0:1775764895.536253:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.536255:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.536263:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.536271:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.536277:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.536285:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.536295:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.536298:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.536301:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.536310:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.536312:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.536314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.536318:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536325:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.536333:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.536335:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.536339:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.536342:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22468, 22468] 00000080:00200000:1.0:1775764895.536352:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.536356:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.536385:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92033024 i_blocks 90112 00000080:00200000:1.0:1775764895.536388:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.536403:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92033024, count: 4096 00000080:00200000:1.0:1775764895.536413:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92033024, bytes: 4096 00000080:00200000:1.0:1775764895.536416:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.536419:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22470, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.536421:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92033024, 92037120) 0 0 00000080:00200000:1.0:1775764895.536423:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.536429:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22469, 22469] 00020000:00200000:1.0:1775764895.536432:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92033024 chunk: [92033024, 92037120) 92037120, 4096 00020000:00200000:1.0:1775764895.536434:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.536445:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22469, 22469] 00000008:00010000:1.0:1775764895.536450:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.536453:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.536461:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536469:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92033024 92037119) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536476:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536484:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536490:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.536498:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92033024, 92037120) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.536503:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22469 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.536526:0:18302:0:(rw26.c:858:ll_write_end()) pos 92033024, len 4096, copied 4096 00000080:00200000:1.0:1775764895.536531:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000eb9559e5[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.536534:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cf80b0db 17ffffc0001021 2:0 ffff90c20b3c8a80 22469 lru 00000080:00200000:1.0:1775764895.536540:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000af4c9643 22469: 1< 0 - - > 2< 92033024 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8630272 - | + - + - > 5< + - + - | 0 - | 1989 - -> 00000080:00200000:1.0:1775764895.536541:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000eb9559e5 00000080:00200000:1.0:1775764895.536542:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.536545:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.536553:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.536560:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.536567:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.536575:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.536586:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.536589:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.536591:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.536601:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.536603:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.536605:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.536609:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536616:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.536624:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.536627:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.536631:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.536634:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22469, 22469] 00000080:00200000:1.0:1775764895.536644:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.536648:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.536677:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92037120 i_blocks 90112 00000080:00200000:1.0:1775764895.536680:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.536697:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92037120, count: 4096 00000080:00200000:1.0:1775764895.536707:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92037120, bytes: 4096 00000080:00200000:1.0:1775764895.536710:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.536714:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22471, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.536715:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92037120, 92041216) 0 0 00000080:00200000:1.0:1775764895.536718:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.536723:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22470, 22470] 00020000:00200000:1.0:1775764895.536726:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92037120 chunk: [92037120, 92041216) 92041216, 4096 00020000:00200000:1.0:1775764895.536728:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.536740:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22470, 22470] 00000008:00010000:1.0:1775764895.536745:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.536747:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.536755:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536765:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92037120 92041215) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536773:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536781:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536787:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.536795:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92037120, 92041216) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.536801:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22470 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.536817:0:18302:0:(rw26.c:858:ll_write_end()) pos 92037120, len 4096, copied 4096 00000080:00200000:1.0:1775764895.536822:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000efc768d0[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.536825:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d73af5b5 17ffffc0001001 3:0 ffff90c20b3c8700 22470 lru 00000080:00200000:1.0:1775764895.536830:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e970e3f8 22470: 1< 0 - - > 2< 92037120 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8626176 - | + - + - > 5< + - + - | 0 - | 1990 - -> 00000080:00200000:1.0:1775764895.536832:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000efc768d0 00000080:00200000:1.0:1775764895.536834:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.536836:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.536843:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.536851:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.536858:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.536865:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.536900:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.536903:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.536906:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.536915:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.536918:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.536921:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.536924:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.536932:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.536942:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.536944:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.536948:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.536950:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22470, 22470] 00000080:00200000:1.0:1775764895.536961:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.536964:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.536993:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92041216 i_blocks 90112 00000080:00200000:1.0:1775764895.536997:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.537013:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92041216, count: 4096 00000080:00200000:1.0:1775764895.537023:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92041216, bytes: 4096 00000080:00200000:1.0:1775764895.537025:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.537029:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22472, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.537031:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92041216, 92045312) 0 0 00000080:00200000:1.0:1775764895.537033:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.537038:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22471, 22471] 00020000:00200000:1.0:1775764895.537041:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92041216 chunk: [92041216, 92045312) 92045312, 4096 00020000:00200000:1.0:1775764895.537043:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.537055:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22471, 22471] 00000008:00010000:1.0:1775764895.537060:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.537062:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.537070:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537078:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92041216 92045311) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537086:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537098:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537105:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.537112:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92041216, 92045312) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.537117:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22471 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.537148:0:18302:0:(rw26.c:858:ll_write_end()) pos 92041216, len 4096, copied 4096 00000080:00200000:1.0:1775764895.537152:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b2ba7584[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.537155:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000416e34cd 17ffffc0001001 3:0 ffff90c20b3c8c40 22471 lru 00000080:00200000:1.0:1775764895.537161:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005f01ff1e 22471: 1< 0 - - > 2< 92041216 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8622080 - | + - + - > 5< + - + - | 0 - | 1991 - -> 00000080:00200000:1.0:1775764895.537163:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b2ba7584 00000080:00200000:1.0:1775764895.537164:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.537166:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.537174:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.537182:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.537189:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.537196:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.537211:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.537215:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.537217:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.537226:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.537229:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.537231:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.537235:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537243:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.537252:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.537255:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.537259:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.537261:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22471, 22471] 00000080:00200000:1.0:1775764895.537272:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.537275:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.537301:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92045312 i_blocks 90112 00000080:00200000:1.0:1775764895.537304:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.537320:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92045312, count: 4096 00000080:00200000:1.0:1775764895.537329:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92045312, bytes: 4096 00000080:00200000:1.0:1775764895.537332:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.537336:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22473, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.537338:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92045312, 92049408) 0 0 00000080:00200000:1.0:1775764895.537340:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.537345:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22472, 22472] 00020000:00200000:1.0:1775764895.537349:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92045312 chunk: [92045312, 92049408) 92049408, 4096 00020000:00200000:1.0:1775764895.537351:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.537362:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22472, 22472] 00000008:00010000:1.0:1775764895.537367:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.537370:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.537377:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537387:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92045312 92049407) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537396:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537403:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537411:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.537420:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92045312, 92049408) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.537425:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22472 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.537441:0:18302:0:(rw26.c:858:ll_write_end()) pos 92045312, len 4096, copied 4096 00000080:00200000:1.0:1775764895.537445:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f6b32822[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.537448:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000bf78ce18 17ffffc0001001 3:0 ffff90c20b3c88c0 22472 lru 00000080:00200000:1.0:1775764895.537454:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a35d4148 22472: 1< 0 - - > 2< 92045312 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8617984 - | + - + - > 5< + - + - | 0 - | 1992 - -> 00000080:00200000:1.0:1775764895.537456:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f6b32822 00000080:00200000:1.0:1775764895.537458:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.537460:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.537468:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.537477:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.537484:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.537492:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.537503:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.537506:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.537508:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.537517:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.537519:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.537522:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.537526:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537534:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.537542:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.537545:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.537549:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.537552:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22472, 22472] 00000080:00200000:1.0:1775764895.537562:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.537565:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.537594:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 92049408 i_blocks 90112 00000080:00200000:1.0:1775764895.537598:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.537613:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 92049408, count: 4096 00000080:00200000:1.0:1775764895.537624:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 92049408, bytes: 4096 00000080:00200000:1.0:1775764895.537627:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.537631:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22474, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.537632:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [92049408, 92053504) 0 0 00000080:00200000:1.0:1775764895.537635:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.537640:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22473, 22473] 00020000:00200000:1.0:1775764895.537643:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 92049408 chunk: [92049408, 92053504) 92053504, 4096 00020000:00200000:1.0:1775764895.537645:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.537657:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22473, 22473] 00000008:00010000:1.0:1775764895.537662:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.537664:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.537672:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537680:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (92049408 92053503) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 13/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537688:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537695:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537702:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.537710:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [92049408, 92053504) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.537715:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22473 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.537730:0:18302:0:(rw26.c:858:ll_write_end()) pos 92049408, len 4096, copied 4096 00000080:00200000:1.0:1775764895.537734:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000445e3e50[3 00000000b99c1f5d 2 1 0000000022fff95b] 00000080:00200000:1.0:1775764895.537737:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b24f8db5 17ffffc0001001 3:0 ffff90c20b3c8b60 22473 lru 00000080:00200000:1.0:1775764895.537742:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000795476ab 22473: 1< 0 - - > 2< 92049408 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 8613888 - | + - + - > 5< + - + - | 0 - | 1993 - -> 00000080:00200000:1.0:1775764895.537745:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000445e3e50 00000080:00200000:1.0:1775764895.537746:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.537748:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.537755:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.537764:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.537771:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.537778:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 0000000059bfa603 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.537790:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.537793:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.537795:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.537804:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.537807:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.537809:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.537814:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.537822:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.537830:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eab78 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.537832:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.537836:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.537838:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22473, 22473] 00000080:00200000:1.0:1775764895.537849:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.537852:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.648101:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93380608 i_blocks 90112 00000080:00200000:1.0:1775764895.648106:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.648139:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93380608, count: 4096 00000080:00200000:1.0:1775764895.648159:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93380608, bytes: 4096 00000080:00200000:1.0:1775764895.648162:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.648166:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22799, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.648167:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93380608, 93384704) 0 0 00000080:00200000:1.0:1775764895.648170:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.648176:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22798, 22798] 00020000:00200000:1.0:1775764895.648180:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93380608 chunk: [93380608, 93384704) 93384704, 4096 00020000:00200000:1.0:1775764895.648183:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.648201:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22798, 22798] 00000008:00010000:1.0:1775764895.648209:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.648212:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.648220:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648230:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93380608 93384703) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648240:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648248:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648254:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.648262:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93380608, 93384704) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.648268:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22798 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.648289:0:18302:0:(rw26.c:858:ll_write_end()) pos 93380608, len 4096, copied 4096 00000080:00200000:1.0:1775764895.648293:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000096b88fe7[3 00000000b99c1f5d 2 1 00000000dc6fd51b] 00000080:00200000:1.0:1775764895.648297:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ee87c849 17ffffc0001001 3:0 ffff90c20b3c8000 22798 lru 00000080:00200000:1.0:1775764895.648302:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000518d20eb 22798: 1< 0 - - > 2< 93380608 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 7258112 - | + - + - > 5< + - + - | 0 - | 2318 - -> 00000080:00200000:1.0:1775764895.648305:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000096b88fe7 00000080:00200000:1.0:1775764895.648306:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.648308:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.648317:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.648325:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.648433:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.648446:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.648459:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.648463:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.648465:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.648477:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.648480:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.648482:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.648487:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648494:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.648503:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.648506:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.648510:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.648514:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22798, 22798] 00000080:00200000:1.0:1775764895.648529:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.648532:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.648648:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93384704 i_blocks 90112 00000080:00200000:1.0:1775764895.648652:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.648671:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93384704, count: 4096 00000080:00200000:1.0:1775764895.648683:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93384704, bytes: 4096 00000080:00200000:1.0:1775764895.648686:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.648689:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22800, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.648691:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93384704, 93388800) 0 0 00000080:00200000:1.0:1775764895.648693:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.648698:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22799, 22799] 00020000:00200000:1.0:1775764895.648701:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93384704 chunk: [93384704, 93388800) 93388800, 4096 00020000:00200000:1.0:1775764895.648704:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.648715:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22799, 22799] 00000008:00010000:1.0:1775764895.648720:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.648723:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.648731:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648740:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93384704 93388799) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648749:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648756:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648762:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.648770:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93384704, 93388800) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.648776:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22799 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.648792:0:18302:0:(rw26.c:858:ll_write_end()) pos 93384704, len 4096, copied 4096 00000080:00200000:1.0:1775764895.648797:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e810585f[3 00000000b99c1f5d 2 1 00000000dc6fd51b] 00000080:00200000:1.0:1775764895.648800:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000050987358 17ffffc0001001 3:0 ffff90c20b3c8ee0 22799 lru 00000080:00200000:1.0:1775764895.648806:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000afcae60c 22799: 1< 0 - - > 2< 93384704 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 7254016 - | + - + - > 5< + - + - | 0 - | 2319 - -> 00000080:00200000:1.0:1775764895.648809:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e810585f 00000080:00200000:1.0:1775764895.648810:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.648812:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.648820:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.648828:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.648836:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.648843:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.648855:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.648857:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.648860:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.648947:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.648952:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.648954:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.648958:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.648967:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.648975:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.648978:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.648982:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.648985:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22799, 22799] 00000080:00200000:1.0:1775764895.648996:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.648999:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.649031:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93388800 i_blocks 90112 00000080:00200000:1.0:1775764895.649035:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.649052:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93388800, count: 4096 00000080:00200000:1.0:1775764895.649064:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93388800, bytes: 4096 00000080:00200000:1.0:1775764895.649066:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.649070:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22801, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.649072:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93388800, 93392896) 0 0 00000080:00200000:1.0:1775764895.649074:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.649079:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22800, 22800] 00020000:00200000:1.0:1775764895.649082:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93388800 chunk: [93388800, 93392896) 93392896, 4096 00020000:00200000:1.0:1775764895.649084:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.649095:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22800, 22800] 00000008:00010000:1.0:1775764895.649100:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.649103:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.649111:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649130:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93388800 93392895) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649139:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649146:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649158:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.649166:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93388800, 93392896) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.649172:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22800 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.649189:0:18302:0:(rw26.c:858:ll_write_end()) pos 93388800, len 4096, copied 4096 00000080:00200000:1.0:1775764895.649194:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d18e4077[3 00000000b99c1f5d 2 1 00000000dc6fd51b] 00000080:00200000:1.0:1775764895.649197:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000da031e9d 17ffffc0001001 3:0 ffff90c20b3c80e0 22800 lru 00000080:00200000:1.0:1775764895.649203:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000854f279d 22800: 1< 0 - - > 2< 93388800 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 7249920 - | + - + - > 5< + - + - | 0 - | 2320 - -> 00000080:00200000:1.0:1775764895.649205:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d18e4077 00000080:00200000:1.0:1775764895.649206:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.649209:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.649216:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.649224:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.649232:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.649239:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.649249:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.649252:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.649255:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.649264:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.649267:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.649269:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.649273:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649280:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.649289:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.649291:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.649295:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.649298:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22800, 22800] 00000080:00200000:1.0:1775764895.649309:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.649312:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.649441:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93392896 i_blocks 90112 00000080:00200000:1.0:1775764895.649446:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.649462:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93392896, count: 4096 00000080:00200000:1.0:1775764895.649473:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93392896, bytes: 4096 00000080:00200000:1.0:1775764895.649476:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.649480:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22802, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.649482:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93392896, 93396992) 0 0 00000080:00200000:1.0:1775764895.649484:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.649489:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22801, 22801] 00020000:00200000:1.0:1775764895.649492:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93392896 chunk: [93392896, 93396992) 93396992, 4096 00020000:00200000:1.0:1775764895.649494:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.649505:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22801, 22801] 00000008:00010000:1.0:1775764895.649511:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.649513:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.649522:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649530:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93392896 93396991) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649538:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649624:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649631:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.649639:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93392896, 93396992) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.649645:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22801 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.649663:0:18302:0:(rw26.c:858:ll_write_end()) pos 93392896, len 4096, copied 4096 00000080:00200000:1.0:1775764895.649668:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000df412be7[3 00000000b99c1f5d 2 1 00000000dc6fd51b] 00000080:00200000:1.0:1775764895.649671:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000aca7ad5f 17ffffc0001001 3:0 ffff90c20b3c81c0 22801 lru 00000080:00200000:1.0:1775764895.649676:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e8d60ad6 22801: 1< 0 - - > 2< 93392896 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 7245824 - | + - + - > 5< + - + - | 0 - | 2321 - -> 00000080:00200000:1.0:1775764895.649678:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000df412be7 00000080:00200000:1.0:1775764895.649679:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.649682:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.649690:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.649698:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.649706:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.649713:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.649724:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.649727:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.649729:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.649739:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.649742:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.649744:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.649749:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649756:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.649765:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.649767:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.649771:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.649774:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22801, 22801] 00000080:00200000:1.0:1775764895.649785:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.649788:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.649819:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93396992 i_blocks 90112 00000080:00200000:1.0:1775764895.649823:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.649840:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93396992, count: 4096 00000080:00200000:1.0:1775764895.649850:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93396992, bytes: 4096 00000080:00200000:1.0:1775764895.649853:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.649857:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22803, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.649858:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93396992, 93401088) 0 0 00000080:00200000:1.0:1775764895.649861:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.649866:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22802, 22802] 00020000:00200000:1.0:1775764895.649896:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93396992 chunk: [93396992, 93401088) 93401088, 4096 00020000:00200000:1.0:1775764895.649899:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.649913:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22802, 22802] 00000008:00010000:1.0:1775764895.649918:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.649921:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.649929:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649938:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93396992 93401087) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649947:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649954:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.649960:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.649968:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93396992, 93401088) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.649973:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22802 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.649990:0:18302:0:(rw26.c:858:ll_write_end()) pos 93396992, len 4096, copied 4096 00000080:00200000:1.0:1775764895.649995:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000048fd6b89[3 00000000b99c1f5d 2 1 00000000dc6fd51b] 00000080:00200000:1.0:1775764895.649997:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000027517be6 17ffffc0001001 3:0 ffff90c20b3c87e0 22802 lru 00000080:00200000:1.0:1775764895.650003:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005eb31ea6 22802: 1< 0 - - > 2< 93396992 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 7241728 - | + - + - > 5< + - + - | 0 - | 2322 - -> 00000080:00200000:1.0:1775764895.650006:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000048fd6b89 00000080:00200000:1.0:1775764895.650007:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.650010:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.650018:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.650026:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.650032:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.650039:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.650051:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.650054:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.650056:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.650067:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.650070:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.650072:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.650075:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650083:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.650091:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.650093:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.650098:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.650100:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22802, 22802] 00000080:00200000:1.0:1775764895.650111:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.650114:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.650154:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93401088 i_blocks 90112 00000080:00200000:1.0:1775764895.650158:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.650174:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93401088, count: 4096 00000080:00200000:1.0:1775764895.650185:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93401088, bytes: 4096 00000080:00200000:1.0:1775764895.650188:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.650191:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22804, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.650193:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93401088, 93405184) 0 0 00000080:00200000:1.0:1775764895.650195:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.650200:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22803, 22803] 00020000:00200000:1.0:1775764895.650203:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93401088 chunk: [93401088, 93405184) 93405184, 4096 00020000:00200000:1.0:1775764895.650205:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.650216:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22803, 22803] 00000008:00010000:1.0:1775764895.650221:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.650224:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.650231:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650240:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93401088 93405183) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650249:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650262:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650268:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.650275:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93401088, 93405184) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.650281:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22803 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.650298:0:18302:0:(rw26.c:858:ll_write_end()) pos 93401088, len 4096, copied 4096 00000080:00200000:1.0:1775764895.650302:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000da42d7a5[3 00000000b99c1f5d 2 1 00000000dc6fd51b] 00000080:00200000:1.0:1775764895.650305:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000171ce3f7 17ffffc0001001 3:0 ffff90c20b3c89a0 22803 lru 00000080:00200000:1.0:1775764895.650311:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004fd65459 22803: 1< 0 - - > 2< 93401088 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 7237632 - | + - + - > 5< + - + - | 0 - | 2323 - -> 00000080:00200000:1.0:1775764895.650313:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000da42d7a5 00000080:00200000:1.0:1775764895.650314:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.650316:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.650324:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.650433:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.650443:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.650451:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.650469:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.650473:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.650475:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.650486:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.650488:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.650491:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.650495:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650503:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.650512:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.650514:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.650519:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.650522:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22803, 22803] 00000080:00200000:1.0:1775764895.650533:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.650536:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.650654:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93405184 i_blocks 90112 00000080:00200000:1.0:1775764895.650658:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.650675:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93405184, count: 4096 00000080:00200000:1.0:1775764895.650686:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93405184, bytes: 4096 00000080:00200000:1.0:1775764895.650689:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.650693:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22805, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.650695:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93405184, 93409280) 0 0 00000080:00200000:1.0:1775764895.650697:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.650703:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22804, 22804] 00020000:00200000:1.0:1775764895.650707:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93405184 chunk: [93405184, 93409280) 93409280, 4096 00020000:00200000:1.0:1775764895.650709:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.650721:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22804, 22804] 00000008:00010000:1.0:1775764895.650726:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.650729:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.650738:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650747:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93405184 93409279) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650756:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650764:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650771:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.650779:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93405184, 93409280) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.650785:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22804 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.650804:0:18302:0:(rw26.c:858:ll_write_end()) pos 93405184, len 4096, copied 4096 00000080:00200000:1.0:1775764895.650808:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006506de2a[3 00000000b99c1f5d 2 1 00000000dc6fd51b] 00000080:00200000:1.0:1775764895.650811:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f7308770 17ffffc0001001 3:0 ffff90c20b3c82a0 22804 lru 00000080:00200000:1.0:1775764895.650816:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000028892aa9 22804: 1< 0 - - > 2< 93405184 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 7233536 - | + - + - > 5< + - + - | 0 - | 2324 - -> 00000080:00200000:1.0:1775764895.650818:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006506de2a 00000080:00200000:1.0:1775764895.650819:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.650821:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.650830:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.650838:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.650845:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.650853:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.650865:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.650867:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.650895:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.650905:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.650908:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.650910:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.650914:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.650921:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.650929:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.650931:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.650935:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.650938:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22804, 22804] 00000080:00200000:1.0:1775764895.650949:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.650952:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.650982:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93409280 i_blocks 90112 00000080:00200000:1.0:1775764895.650985:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.651001:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93409280, count: 4096 00000080:00200000:1.0:1775764895.651011:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93409280, bytes: 4096 00000080:00200000:1.0:1775764895.651013:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.651016:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22806, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.651018:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93409280, 93413376) 0 0 00000080:00200000:1.0:1775764895.651020:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.651025:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22805, 22805] 00020000:00200000:1.0:1775764895.651028:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93409280 chunk: [93409280, 93413376) 93413376, 4096 00020000:00200000:1.0:1775764895.651030:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.651040:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22805, 22805] 00000008:00010000:1.0:1775764895.651046:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.651048:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.651055:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.651064:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93409280 93413375) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.651071:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.651078:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.651085:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.651093:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93409280, 93413376) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.651098:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22805 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.651115:0:18302:0:(rw26.c:858:ll_write_end()) pos 93409280, len 4096, copied 4096 00000080:00200000:1.0:1775764895.651129:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000667285a4[3 00000000b99c1f5d 2 1 00000000dc6fd51b] 00000080:00200000:1.0:1775764895.651131:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000079096ab9 17ffffc0001001 3:0 ffff90c20b3c8d20 22805 lru 00000080:00200000:1.0:1775764895.651136:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ab16927e 22805: 1< 0 - - > 2< 93409280 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 7229440 - | + - + - > 5< + - + - | 0 - | 2325 - -> 00000080:00200000:1.0:1775764895.651138:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000667285a4 00000080:00200000:1.0:1775764895.651139:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.651141:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.651149:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.651156:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.651163:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.651170:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.651183:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.651186:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.651188:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.651198:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.651200:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.651202:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.651206:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.651213:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.651221:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.651223:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.651227:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.651229:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22805, 22805] 00000080:00200000:1.0:1775764895.651240:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.651243:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.651272:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93413376 i_blocks 90112 00000080:00200000:1.0:1775764895.651276:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.651291:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93413376, count: 4096 00000080:00200000:1.0:1775764895.651301:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93413376, bytes: 4096 00000080:00200000:1.0:1775764895.651304:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.651307:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22807, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.651309:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93413376, 93417472) 0 0 00000080:00200000:1.0:1775764895.651311:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.651316:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22806, 22806] 00020000:00200000:1.0:1775764895.651319:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93413376 chunk: [93413376, 93417472) 93417472, 4096 00020000:00200000:1.0:1775764895.651321:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.651431:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22806, 22806] 00000008:00010000:1.0:1775764895.651438:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.651440:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.651448:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.651458:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93413376 93417471) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.651467:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.651475:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.651481:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.651488:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93413376, 93417472) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.651493:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22806 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.651510:0:18302:0:(rw26.c:858:ll_write_end()) pos 93413376, len 4096, copied 4096 00000080:00200000:1.0:1775764895.651514:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000008f4b458[3 00000000b99c1f5d 2 1 00000000dc6fd51b] 00000080:00200000:1.0:1775764895.651516:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000505b2d11 17ffffc0001001 3:0 ffff90c20b3c8380 22806 lru 00000080:00200000:1.0:1775764895.651522:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cf5eb328 22806: 1< 0 - - > 2< 93413376 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 7225344 - | + - + - > 5< + - + - | 0 - | 2326 - -> 00000080:00200000:1.0:1775764895.651524:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000008f4b458 00000080:00200000:1.0:1775764895.651525:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.651527:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.651534:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.651541:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a5d09451 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.652039:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.652054:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.652070:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.652074:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.652077:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.652091:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.652093:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.652096:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.652101:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.652109:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.652127:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea3f8 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.652130:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.652135:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.652138:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22806, 22806] 00000080:00200000:1.0:1775764895.655394:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.655404:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.735829:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.735890:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.735907:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a3bd5654 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.735917:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.735926:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.735958:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.735975:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.735980:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.736011:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f7128 (ffff90c20361ccf8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.736014:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f7128 (ffff90c20361ccf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.736018:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a5016b0 (ffff90c20361c128/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.736025:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.736035:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.736050:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f7128 (ffff90c20361ccf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.736054:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a5016b0 (ffff90c20361ccf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.736063:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.736070:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22910, 22910] 00000080:00200000:1.0:1775764895.736104:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.736108:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.736197:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93843456 i_blocks 90112 00000080:00200000:1.0:1775764895.736203:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.736243:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93843456, count: 4096 00000080:00200000:1.0:1775764895.736257:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93843456, bytes: 4096 00000080:00200000:1.0:1775764895.736262:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.736269:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22912, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.736271:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93843456, 93847552) 0 0 00000080:00200000:1.0:1775764895.736275:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.736288:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22911, 22911] 00020000:00200000:1.0:1775764895.736295:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93843456 chunk: [93843456, 93847552) 93847552, 4096 00020000:00200000:1.0:1775764895.736299:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.736322:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22911, 22911] 00000008:00010000:1.0:1775764895.737493:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.737503:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.737543:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.737563:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93843456 93847551) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.737579:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.737590:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.737598:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.737614:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93843456, 93847552) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.737631:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22911 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.737686:0:18302:0:(rw26.c:858:ll_write_end()) pos 93843456, len 4096, copied 4096 00000080:00200000:1.0:1775764895.737696:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000254bc3f5[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.737699:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009bb327c8 17ffffc0001001 3:0 ffff90c20b3c8620 22911 lru 00000080:00200000:1.0:1775764895.737708:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003d511252 22911: 1< 0 - - > 2< 93843456 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6795264 - | + - + - > 5< + - + - | 0 - | 2431 - -> 00000080:00200000:1.0:1775764895.737711:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000254bc3f5 00000080:00200000:1.0:1775764895.737712:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.737716:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.738241:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.738253:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.738261:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.738270:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.738296:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.738301:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.738304:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.738332:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.738335:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.738338:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.738343:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.738352:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.738366:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.738369:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.738377:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.738386:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22911, 22911] 00000080:00200000:1.0:1775764895.738421:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.738426:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.738493:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93847552 i_blocks 90112 00000080:00200000:1.0:1775764895.738498:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.738667:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93847552, count: 4096 00000080:00200000:1.0:1775764895.738688:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93847552, bytes: 4096 00000080:00200000:1.0:1775764895.738693:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.738698:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22913, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.738701:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93847552, 93851648) 0 0 00000080:00200000:1.0:1775764895.738705:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.738720:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22912, 22912] 00020000:00200000:1.0:1775764895.738726:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93847552 chunk: [93847552, 93851648) 93851648, 4096 00020000:00200000:1.0:1775764895.738731:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.738752:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22912, 22912] 00000008:00010000:1.0:1775764895.738763:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.738767:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.738781:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.738792:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93847552 93851647) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.738801:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.738815:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.738822:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.738831:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93847552, 93851648) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.738839:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22912 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.738908:0:18302:0:(rw26.c:858:ll_write_end()) pos 93847552, len 4096, copied 4096 00000080:00200000:1.0:1775764895.738914:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c7312db5[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.738917:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000768ce5fe 17ffffc0001021 2:0 ffff90c20b3c8e00 22912 lru 00000080:00200000:1.0:1775764895.738924:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001d81db50 22912: 1< 0 - - > 2< 93847552 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6791168 - | + - + - > 5< + - + - | 0 - | 2432 - -> 00000080:00200000:1.0:1775764895.738926:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c7312db5 00000080:00200000:1.0:1775764895.738927:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.738930:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.738940:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.738949:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.738956:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.738977:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.738998:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.739001:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.739004:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.739017:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.739020:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.739023:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.739027:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.739035:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.739045:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.739048:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.739054:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.739057:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22912, 22912] 00000080:00200000:1.0:1775764895.739068:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.739071:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.739108:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93851648 i_blocks 90112 00000080:00200000:1.0:1775764895.739112:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.739147:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93851648, count: 4096 00000080:00200000:1.0:1775764895.739157:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93851648, bytes: 4096 00000080:00200000:1.0:1775764895.739160:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.739164:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22914, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.739165:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93851648, 93855744) 0 0 00000080:00200000:1.0:1775764895.739168:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.739174:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22913, 22913] 00020000:00200000:1.0:1775764895.739177:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93851648 chunk: [93851648, 93855744) 93855744, 4096 00020000:00200000:1.0:1775764895.739179:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.739189:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22913, 22913] 00000008:00010000:1.0:1775764895.739195:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.739198:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.739206:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.739216:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93851648 93855743) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.739225:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.739233:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.739240:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.739248:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93851648, 93855744) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.739254:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22913 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.739282:0:18302:0:(rw26.c:858:ll_write_end()) pos 93851648, len 4096, copied 4096 00000080:00200000:1.0:1775764895.739287:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000215d2b8e[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.739290:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000088b9b495 17ffffc0001001 3:0 ffff90c20b3c91c0 22913 lru 00000080:00200000:1.0:1775764895.739296:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a6bdb53c 22913: 1< 0 - - > 2< 93851648 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6787072 - | + - + - > 5< + - + - | 0 - | 2433 - -> 00000080:00200000:1.0:1775764895.739298:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000215d2b8e 00000080:00200000:1.0:1775764895.739300:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.739302:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.739310:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.739318:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.739326:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.739334:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.739346:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.739350:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.739352:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.739362:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.739364:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.739367:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.739371:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.739379:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.739387:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.739389:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.739393:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.739397:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22913, 22913] 00000080:00200000:1.0:1775764895.739408:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.739411:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.739442:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93855744 i_blocks 90112 00000080:00200000:1.0:1775764895.739446:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.739463:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93855744, count: 4096 00000080:00200000:1.0:1775764895.739474:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93855744, bytes: 4096 00000080:00200000:1.0:1775764895.739477:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.739481:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22915, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.739483:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93855744, 93859840) 0 0 00000080:00200000:1.0:1775764895.739485:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.739491:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22914, 22914] 00020000:00200000:1.0:1775764895.739494:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93855744 chunk: [93855744, 93859840) 93859840, 4096 00020000:00200000:1.0:1775764895.739496:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.739507:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22914, 22914] 00000008:00010000:1.0:1775764895.739513:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.739515:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.739536:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.741052:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93855744 93859839) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.741071:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.741080:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.741087:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.741104:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93855744, 93859840) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.741132:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22914 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.741179:0:18302:0:(rw26.c:858:ll_write_end()) pos 93855744, len 4096, copied 4096 00000080:00200000:1.0:1775764895.741187:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d60f6386[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.741191:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004bfff264 17ffffc0001001 3:0 ffff90c20b3c97e0 22914 lru 00000080:00200000:1.0:1775764895.741198:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000085bd6b04 22914: 1< 0 - - > 2< 93855744 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6782976 - | + - + - > 5< + - + - | 0 - | 2434 - -> 00000080:00200000:1.0:1775764895.741201:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d60f6386 00000080:00200000:1.0:1775764895.741203:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.741207:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.741226:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.741236:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.741244:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.741252:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.741274:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.741279:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.741282:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.741306:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.741309:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.741311:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.741317:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.741326:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.741339:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.741342:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.741350:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.741357:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22914, 22914] 00000080:00200000:1.0:1775764895.741386:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.741391:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.741450:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93859840 i_blocks 90112 00000080:00200000:1.0:1775764895.741457:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.741488:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93859840, count: 4096 00000080:00200000:1.0:1775764895.741503:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93859840, bytes: 4096 00000080:00200000:1.0:1775764895.741507:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.741513:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22916, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.741515:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93859840, 93863936) 0 0 00000080:00200000:1.0:1775764895.742257:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.742279:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22915, 22915] 00020000:00200000:1.0:1775764895.742288:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93859840 chunk: [93859840, 93863936) 93863936, 4096 00020000:00200000:1.0:1775764895.742292:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.742326:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22915, 22915] 00000008:00010000:1.0:1775764895.742439:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.742445:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.742468:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.742482:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93859840 93863935) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.742494:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.742502:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.742510:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.742682:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93859840, 93863936) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.742695:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22915 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.742745:0:18302:0:(rw26.c:858:ll_write_end()) pos 93859840, len 4096, copied 4096 00000080:00200000:1.0:1775764895.742753:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000032ffe3d4[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.742757:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000909491d2 17ffffc0001001 3:0 ffff90c20b3c99a0 22915 lru 00000080:00200000:1.0:1775764895.742763:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ebcebc33 22915: 1< 0 - - > 2< 93859840 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6778880 - | + - + - > 5< + - + - | 0 - | 2435 - -> 00000080:00200000:1.0:1775764895.742765:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000032ffe3d4 00000080:00200000:1.0:1775764895.742767:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.742771:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.742788:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.742798:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.742806:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.742820:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.742839:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.742843:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.742845:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.742865:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.742906:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.742910:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.742915:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.742924:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.742937:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.742940:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.742946:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.742951:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22915, 22915] 00000080:00200000:1.0:1775764895.742971:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.742975:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.743025:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93863936 i_blocks 90112 00000080:00200000:1.0:1775764895.743031:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.743063:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93863936, count: 4096 00000080:00200000:1.0:1775764895.743079:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93863936, bytes: 4096 00000080:00200000:1.0:1775764895.743084:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.743089:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22917, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.743092:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93863936, 93868032) 0 0 00000080:00200000:1.0:1775764895.743095:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.743104:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22916, 22916] 00020000:00200000:1.0:1775764895.743107:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93863936 chunk: [93863936, 93868032) 93868032, 4096 00020000:00200000:1.0:1775764895.743110:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.743135:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22916, 22916] 00000008:00010000:1.0:1775764895.743142:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.743146:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.743158:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743168:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93863936 93868031) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743177:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743185:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743193:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.743202:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93863936, 93868032) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.743207:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22916 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.743224:0:18302:0:(rw26.c:858:ll_write_end()) pos 93863936, len 4096, copied 4096 00000080:00200000:1.0:1775764895.743229:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000002fc1eb2[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.743232:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009cc91de5 17ffffc0001001 3:0 ffff90c20b3c92a0 22916 lru 00000080:00200000:1.0:1775764895.743237:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000619db7fb 22916: 1< 0 - - > 2< 93863936 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6774784 - | + - + - > 5< + - + - | 0 - | 2436 - -> 00000080:00200000:1.0:1775764895.743240:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000002fc1eb2 00000080:00200000:1.0:1775764895.743241:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.743244:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.743252:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.743260:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.743268:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.743276:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.743287:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.743290:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.743292:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.743302:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.743305:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.743307:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.743310:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743318:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.743327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.743329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.743346:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.743349:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22916, 22916] 00000080:00200000:1.0:1775764895.743360:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.743364:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.743394:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93868032 i_blocks 90112 00000080:00200000:1.0:1775764895.743399:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.743415:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93868032, count: 4096 00000080:00200000:1.0:1775764895.743425:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93868032, bytes: 4096 00000080:00200000:1.0:1775764895.743427:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.743431:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22918, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.743432:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93868032, 93872128) 0 0 00000080:00200000:1.0:1775764895.743435:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.743440:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22917, 22917] 00020000:00200000:1.0:1775764895.743442:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93868032 chunk: [93868032, 93872128) 93872128, 4096 00020000:00200000:1.0:1775764895.743445:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.743455:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22917, 22917] 00000008:00010000:1.0:1775764895.743460:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.743463:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.743471:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743480:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93868032 93872127) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743489:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743496:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743508:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.743515:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93868032, 93872128) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.743532:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22917 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.743550:0:18302:0:(rw26.c:858:ll_write_end()) pos 93868032, len 4096, copied 4096 00000080:00200000:1.0:1775764895.743555:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003d006b92[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.743558:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000051457131 17ffffc0001001 3:0 ffff90c20b3c9d20 22917 lru 00000080:00200000:1.0:1775764895.743563:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b4bc25fd 22917: 1< 0 - - > 2< 93868032 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6770688 - | + - + - > 5< + - + - | 0 - | 2437 - -> 00000080:00200000:1.0:1775764895.743565:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003d006b92 00000080:00200000:1.0:1775764895.743566:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.743569:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.743577:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.743584:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.743591:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.743597:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.743608:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.743611:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.743614:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.743624:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.743626:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.743628:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.743632:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743640:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.743648:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.743650:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.743654:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.743656:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22917, 22917] 00000080:00200000:1.0:1775764895.743666:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.743669:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.743699:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93872128 i_blocks 90112 00000080:00200000:1.0:1775764895.743703:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.743719:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93872128, count: 4096 00000080:00200000:1.0:1775764895.743729:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93872128, bytes: 4096 00000080:00200000:1.0:1775764895.743731:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.743735:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22919, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.743737:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93872128, 93876224) 0 0 00000080:00200000:1.0:1775764895.743739:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.743744:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22918, 22918] 00020000:00200000:1.0:1775764895.743747:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93872128 chunk: [93872128, 93876224) 93876224, 4096 00020000:00200000:1.0:1775764895.743750:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.743761:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22918, 22918] 00000008:00010000:1.0:1775764895.743766:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.743769:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.743777:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743785:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93872128 93876223) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743792:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743799:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743806:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.743814:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93872128, 93876224) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.743819:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22918 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.743835:0:18302:0:(rw26.c:858:ll_write_end()) pos 93872128, len 4096, copied 4096 00000080:00200000:1.0:1775764895.743840:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008f6763d1[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.743843:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000123b9e74 17ffffc0001001 3:0 ffff90c20b3c9380 22918 lru 00000080:00200000:1.0:1775764895.743848:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f7f10878 22918: 1< 0 - - > 2< 93872128 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6766592 - | + - + - > 5< + - + - | 0 - | 2438 - -> 00000080:00200000:1.0:1775764895.743850:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008f6763d1 00000080:00200000:1.0:1775764895.743851:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.743853:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.743861:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.743893:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.743901:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.743909:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.743920:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.743924:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.743926:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.743935:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.743937:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.743940:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.743944:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.743952:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.743959:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.743961:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.743965:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.743968:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22918, 22918] 00000080:00200000:1.0:1775764895.743979:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.743982:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.744011:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93876224 i_blocks 90112 00000080:00200000:1.0:1775764895.744014:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.744030:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93876224, count: 4096 00000080:00200000:1.0:1775764895.744041:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93876224, bytes: 4096 00000080:00200000:1.0:1775764895.744044:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.744048:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22920, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.744049:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93876224, 93880320) 0 0 00000080:00200000:1.0:1775764895.744052:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.744057:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22919, 22919] 00020000:00200000:1.0:1775764895.744060:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93876224 chunk: [93876224, 93880320) 93880320, 4096 00020000:00200000:1.0:1775764895.744062:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.744074:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22919, 22919] 00000008:00010000:1.0:1775764895.744078:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.744081:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.744088:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744097:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93876224 93880319) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744105:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744113:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744133:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.744142:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93876224, 93880320) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.744148:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22919 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.744165:0:18302:0:(rw26.c:858:ll_write_end()) pos 93876224, len 4096, copied 4096 00000080:00200000:1.0:1775764895.744169:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000626cbda0[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.744172:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000013fb9b90 17ffffc0001001 3:0 ffff90c20b3c9620 22919 lru 00000080:00200000:1.0:1775764895.744178:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e6e1d07c 22919: 1< 0 - - > 2< 93876224 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6762496 - | + - + - > 5< + - + - | 0 - | 2439 - -> 00000080:00200000:1.0:1775764895.744180:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000626cbda0 00000080:00200000:1.0:1775764895.744181:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.744183:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.744191:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.744199:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.744206:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.744214:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.744225:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.744227:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.744229:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.744238:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.744240:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.744243:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.744246:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744254:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.744262:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.744264:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.744269:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.744272:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22919, 22919] 00000080:00200000:1.0:1775764895.744282:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.744285:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.744314:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93880320 i_blocks 90112 00000080:00200000:1.0:1775764895.744317:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.744333:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93880320, count: 4096 00000080:00200000:1.0:1775764895.744343:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93880320, bytes: 4096 00000080:00200000:1.0:1775764895.744346:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.744350:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22921, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.744352:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93880320, 93884416) 0 0 00000080:00200000:1.0:1775764895.744354:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.744360:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22920, 22920] 00020000:00200000:1.0:1775764895.744363:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93880320 chunk: [93880320, 93884416) 93884416, 4096 00020000:00200000:1.0:1775764895.744365:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.744376:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22920, 22920] 00000008:00010000:1.0:1775764895.744381:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.744384:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.744392:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744401:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93880320 93884415) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744411:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744424:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744431:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.744439:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93880320, 93884416) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.744444:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22920 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.744464:0:18302:0:(rw26.c:858:ll_write_end()) pos 93880320, len 4096, copied 4096 00000080:00200000:1.0:1775764895.744468:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004df58adc[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.744471:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000040089b91 17ffffc0001001 3:0 ffff90c20b3c9e00 22920 lru 00000080:00200000:1.0:1775764895.744476:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ae20ae07 22920: 1< 0 - - > 2< 93880320 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6758400 - | + - + - > 5< + - + - | 0 - | 2440 - -> 00000080:00200000:1.0:1775764895.744478:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004df58adc 00000080:00200000:1.0:1775764895.744479:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.744481:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.744489:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.744497:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.744504:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.744511:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.744528:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.744531:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.744534:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.744544:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.744547:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.744549:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.744552:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744561:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.744569:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.744571:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.744575:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.744578:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22920, 22920] 00000080:00200000:1.0:1775764895.744588:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.744591:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.744621:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93884416 i_blocks 90112 00000080:00200000:1.0:1775764895.744625:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.744641:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93884416, count: 4096 00000080:00200000:1.0:1775764895.744652:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93884416, bytes: 4096 00000080:00200000:1.0:1775764895.744655:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.744659:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22922, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.744661:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93884416, 93888512) 0 0 00000080:00200000:1.0:1775764895.744663:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.744668:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22921, 22921] 00020000:00200000:1.0:1775764895.744671:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93884416 chunk: [93884416, 93888512) 93888512, 4096 00020000:00200000:1.0:1775764895.744674:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.744686:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22921, 22921] 00000008:00010000:1.0:1775764895.744691:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.744694:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.744702:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744711:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93884416 93888511) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744721:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744729:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.744737:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.744745:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93884416, 93888512) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.744750:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22921 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.745415:0:18302:0:(rw26.c:858:ll_write_end()) pos 93884416, len 4096, copied 4096 00000080:00200000:1.0:1775764895.745424:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f3cd3b58[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.745428:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000054a46d7b 17ffffc0001001 3:0 ffff90c20b3c9540 22921 lru 00000080:00200000:1.0:1775764895.745434:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000168d8961 22921: 1< 0 - - > 2< 93884416 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6754304 - | + - + - > 5< + - + - | 0 - | 2441 - -> 00000080:00200000:1.0:1775764895.745436:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f3cd3b58 00000080:00200000:1.0:1775764895.745437:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.745440:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.745457:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.745466:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.745472:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.745479:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.745492:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.745496:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.745498:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.745515:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.745517:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.745520:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.745524:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.745532:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.745543:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.745545:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.745551:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.745556:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22921, 22921] 00000080:00200000:1.0:1775764895.745574:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.745577:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.745624:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93888512 i_blocks 90112 00000080:00200000:1.0:1775764895.745630:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.745655:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93888512, count: 4096 00000080:00200000:1.0:1775764895.745669:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93888512, bytes: 4096 00000080:00200000:1.0:1775764895.745673:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.745677:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22923, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.745679:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93888512, 93892608) 0 0 00000080:00200000:1.0:1775764895.745682:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.745691:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22922, 22922] 00020000:00200000:1.0:1775764895.745696:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93888512 chunk: [93888512, 93892608) 93892608, 4096 00020000:00200000:1.0:1775764895.745699:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.745714:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22922, 22922] 00000008:00010000:1.0:1775764895.745723:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.745727:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.745739:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.745748:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93888512 93892607) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.745757:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.745764:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.745770:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.745779:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93888512, 93892608) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.745788:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22922 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.745816:0:18302:0:(rw26.c:858:ll_write_end()) pos 93888512, len 4096, copied 4096 00000080:00200000:1.0:1775764895.745820:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007319e08a[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.745823:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ccdd8003 17ffffc0001001 3:0 ffff90c20b3c9460 22922 lru 00000080:00200000:1.0:1775764895.745829:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000bc9ada66 22922: 1< 0 - - > 2< 93888512 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6750208 - | + - + - > 5< + - + - | 0 - | 2442 - -> 00000080:00200000:1.0:1775764895.745831:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007319e08a 00000080:00200000:1.0:1775764895.745832:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.745834:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.745842:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.745851:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.745857:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.745865:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.746107:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.746111:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.746114:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.746138:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.746140:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.746143:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.746147:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746155:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.746162:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.746165:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.746169:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.746172:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22922, 22922] 00000080:00200000:1.0:1775764895.746184:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.746187:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.746221:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93892608 i_blocks 90112 00000080:00200000:1.0:1775764895.746225:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.746242:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93892608, count: 4096 00000080:00200000:1.0:1775764895.746253:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93892608, bytes: 4096 00000080:00200000:1.0:1775764895.746256:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.746259:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22924, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.746261:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93892608, 93896704) 0 0 00000080:00200000:1.0:1775764895.746263:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.746269:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22923, 22923] 00020000:00200000:1.0:1775764895.746272:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93892608 chunk: [93892608, 93896704) 93896704, 4096 00020000:00200000:1.0:1775764895.746274:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.746286:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22923, 22923] 00000008:00010000:1.0:1775764895.746291:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.746294:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.746303:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746312:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93892608 93896703) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746322:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746415:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746422:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.746431:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93892608, 93896704) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.746437:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22923 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.746457:0:18302:0:(rw26.c:858:ll_write_end()) pos 93892608, len 4096, copied 4096 00000080:00200000:1.0:1775764895.746462:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e4afdd1f[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.746465:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009ab332d3 17ffffc0001001 3:0 ffff90c20b3c9a80 22923 lru 00000080:00200000:1.0:1775764895.746470:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000734e8e58 22923: 1< 0 - - > 2< 93892608 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6746112 - | + - + - > 5< + - + - | 0 - | 2443 - -> 00000080:00200000:1.0:1775764895.746472:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e4afdd1f 00000080:00200000:1.0:1775764895.746473:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.746476:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.746484:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.746492:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.746499:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.746512:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.746577:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.746581:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.746583:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.746593:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.746596:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.746598:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.746603:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746611:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.746620:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.746622:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.746627:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.746630:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22923, 22923] 00000080:00200000:1.0:1775764895.746641:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.746644:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.746677:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93896704 i_blocks 90112 00000080:00200000:1.0:1775764895.746681:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.746697:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93896704, count: 4096 00000080:00200000:1.0:1775764895.746709:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93896704, bytes: 4096 00000080:00200000:1.0:1775764895.746711:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.746716:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22925, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.746717:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93896704, 93900800) 0 0 00000080:00200000:1.0:1775764895.746720:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.746726:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22924, 22924] 00020000:00200000:1.0:1775764895.746729:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93896704 chunk: [93896704, 93900800) 93900800, 4096 00020000:00200000:1.0:1775764895.746731:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.746744:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22924, 22924] 00000008:00010000:1.0:1775764895.746749:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.746752:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.746760:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746768:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93896704 93900799) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746776:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746783:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746790:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.746799:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93896704, 93900800) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.746804:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22924 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.746820:0:18302:0:(rw26.c:858:ll_write_end()) pos 93896704, len 4096, copied 4096 00000080:00200000:1.0:1775764895.746824:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006cc7d4f4[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.746827:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000bdbbdffb 17ffffc0001001 3:0 ffff90c20b3c9700 22924 lru 00000080:00200000:1.0:1775764895.746833:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f0d23675 22924: 1< 0 - - > 2< 93896704 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6742016 - | + - + - > 5< + - + - | 0 - | 2444 - -> 00000080:00200000:1.0:1775764895.746835:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006cc7d4f4 00000080:00200000:1.0:1775764895.746836:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.746839:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.746846:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.746854:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.746861:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.746891:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.746904:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.746907:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.746909:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.746919:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.746921:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.746923:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.746927:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.746934:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.746941:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.746944:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.746948:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.746950:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22924, 22924] 00000080:00200000:1.0:1775764895.746960:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.746963:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.746991:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93900800 i_blocks 90112 00000080:00200000:1.0:1775764895.746994:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.747010:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93900800, count: 4096 00000080:00200000:1.0:1775764895.747020:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93900800, bytes: 4096 00000080:00200000:1.0:1775764895.747022:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.747026:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22926, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.747028:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93900800, 93904896) 0 0 00000080:00200000:1.0:1775764895.747030:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.747036:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22925, 22925] 00020000:00200000:1.0:1775764895.747039:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93900800 chunk: [93900800, 93904896) 93904896, 4096 00020000:00200000:1.0:1775764895.747041:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.747051:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22925, 22925] 00000008:00010000:1.0:1775764895.747056:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.747059:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.747067:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747077:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93900800 93904895) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747085:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747092:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747105:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.747112:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93900800, 93904896) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.747128:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22925 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.747145:0:18302:0:(rw26.c:858:ll_write_end()) pos 93900800, len 4096, copied 4096 00000080:00200000:1.0:1775764895.747149:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b71fe185[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.747152:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d1f6375d 17ffffc0001001 3:0 ffff90c20b3c9c40 22925 lru 00000080:00200000:1.0:1775764895.747157:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b5265b53 22925: 1< 0 - - > 2< 93900800 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6737920 - | + - + - > 5< + - + - | 0 - | 2445 - -> 00000080:00200000:1.0:1775764895.747160:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b71fe185 00000080:00200000:1.0:1775764895.747161:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.747163:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.747171:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.747178:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.747184:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.747191:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.747202:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.747205:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.747207:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.747217:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.747220:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.747222:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.747226:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747234:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.747242:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.747245:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.747249:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.747252:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22925, 22925] 00000080:00200000:1.0:1775764895.747262:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.747266:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.747295:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93904896 i_blocks 90112 00000080:00200000:1.0:1775764895.747300:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.747316:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93904896, count: 4096 00000080:00200000:1.0:1775764895.747327:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93904896, bytes: 4096 00000080:00200000:1.0:1775764895.747428:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.747433:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22927, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.747435:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93904896, 93908992) 0 0 00000080:00200000:1.0:1775764895.747437:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.747443:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22926, 22926] 00020000:00200000:1.0:1775764895.747446:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93904896 chunk: [93904896, 93908992) 93908992, 4096 00020000:00200000:1.0:1775764895.747448:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.747461:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22926, 22926] 00000008:00010000:1.0:1775764895.747467:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.747470:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.747479:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747487:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93904896 93908991) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747496:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747504:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747511:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.747674:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93904896, 93908992) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.747682:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22926 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.747702:0:18302:0:(rw26.c:858:ll_write_end()) pos 93904896, len 4096, copied 4096 00000080:00200000:1.0:1775764895.747706:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e7c3576b[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.747709:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004f10fc63 17ffffc0001001 3:0 ffff90c20b3c98c0 22926 lru 00000080:00200000:1.0:1775764895.747714:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000051562c84 22926: 1< 0 - - > 2< 93904896 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6733824 - | + - + - > 5< + - + - | 0 - | 2446 - -> 00000080:00200000:1.0:1775764895.747716:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e7c3576b 00000080:00200000:1.0:1775764895.747717:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.747720:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.747727:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.747736:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.747743:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.747750:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.747763:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.747766:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.747768:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.747780:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.747783:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.747785:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.747789:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747797:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.747806:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.747808:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.747813:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.747816:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22926, 22926] 00000080:00200000:1.0:1775764895.747827:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.747830:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.747862:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93908992 i_blocks 90112 00000080:00200000:1.0:1775764895.747866:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.747910:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93908992, count: 4096 00000080:00200000:1.0:1775764895.747923:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93908992, bytes: 4096 00000080:00200000:1.0:1775764895.747925:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.747929:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22928, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.747931:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93908992, 93913088) 0 0 00000080:00200000:1.0:1775764895.747933:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.747938:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22927, 22927] 00020000:00200000:1.0:1775764895.747941:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93908992 chunk: [93908992, 93913088) 93913088, 4096 00020000:00200000:1.0:1775764895.747944:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.747955:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22927, 22927] 00000008:00010000:1.0:1775764895.747961:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.747963:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.747971:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747981:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93908992 93913087) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747989:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.747997:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.748003:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.748012:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93908992, 93913088) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.748017:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22927 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.748041:0:18302:0:(rw26.c:858:ll_write_end()) pos 93908992, len 4096, copied 4096 00000080:00200000:1.0:1775764895.748045:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a4d33b3d[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.748048:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000084e01d99 17ffffc0001021 2:0 ffff90c20b3c9b60 22927 lru 00000080:00200000:1.0:1775764895.748054:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009f88c196 22927: 1< 0 - - > 2< 93908992 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6729728 - | + - + - > 5< + - + - | 0 - | 2447 - -> 00000080:00200000:1.0:1775764895.748056:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a4d33b3d 00000080:00200000:1.0:1775764895.748057:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.748059:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.748067:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.748075:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.748082:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.748090:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.748101:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.748103:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.748106:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.748116:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.748130:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.748133:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.748136:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.748144:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.748153:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.748155:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.748159:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.748162:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22927, 22927] 00000080:00200000:1.0:1775764895.748172:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.748175:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.748203:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93913088 i_blocks 90112 00000080:00200000:1.0:1775764895.748207:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.748222:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93913088, count: 4096 00000080:00200000:1.0:1775764895.748232:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93913088, bytes: 4096 00000080:00200000:1.0:1775764895.748235:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.748238:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22929, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.748240:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93913088, 93917184) 0 0 00000080:00200000:1.0:1775764895.748242:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.748247:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22928, 22928] 00020000:00200000:1.0:1775764895.748251:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93913088 chunk: [93913088, 93917184) 93917184, 4096 00020000:00200000:1.0:1775764895.748253:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.748264:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22928, 22928] 00000008:00010000:1.0:1775764895.748270:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.748272:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.748280:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.748289:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93913088 93917183) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.748297:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.748308:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.748314:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.748321:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93913088, 93917184) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.748327:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22928 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.748683:0:18302:0:(rw26.c:858:ll_write_end()) pos 93913088, len 4096, copied 4096 00000080:00200000:1.0:1775764895.748689:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009d2a1831[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.748692:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007ff9a50a 17ffffc0001001 3:0 ffff90c20b3c9000 22928 lru 00000080:00200000:1.0:1775764895.748698:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004b8061f6 22928: 1< 0 - - > 2< 93913088 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6725632 - | + - + - > 5< + - + - | 0 - | 2448 - -> 00000080:00200000:1.0:1775764895.748700:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009d2a1831 00000080:00200000:1.0:1775764895.748701:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.748704:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.748714:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.748722:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.748730:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.748738:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.748754:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.748758:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.748760:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.748772:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.748775:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.748777:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.748781:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.748789:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.748798:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.748801:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.748806:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.748809:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22928, 22928] 00000080:00200000:1.0:1775764895.748822:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.748825:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.748864:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93917184 i_blocks 90112 00000080:00200000:1.0:1775764895.748892:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.748913:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93917184, count: 4096 00000080:00200000:1.0:1775764895.748925:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93917184, bytes: 4096 00000080:00200000:1.0:1775764895.748928:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.748933:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22930, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.748934:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93917184, 93921280) 0 0 00000080:00200000:1.0:1775764895.748936:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.748944:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22929, 22929] 00020000:00200000:1.0:1775764895.748948:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93917184 chunk: [93917184, 93921280) 93921280, 4096 00020000:00200000:1.0:1775764895.748950:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.748963:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22929, 22929] 00000008:00010000:1.0:1775764895.748969:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.748973:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.748983:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.748992:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93917184 93921279) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.749000:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.749008:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.749015:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.749023:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93917184, 93921280) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.749029:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22929 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.749049:0:18302:0:(rw26.c:858:ll_write_end()) pos 93917184, len 4096, copied 4096 00000080:00200000:1.0:1775764895.749053:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b48a5c6a[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.749056:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009f69aab3 17ffffc0001001 3:0 ffff90c20b3c9ee0 22929 lru 00000080:00200000:1.0:1775764895.749062:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000051d955ca 22929: 1< 0 - - > 2< 93917184 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6721536 - | + - + - > 5< + - + - | 0 - | 2449 - -> 00000080:00200000:1.0:1775764895.749064:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b48a5c6a 00000080:00200000:1.0:1775764895.749066:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.749068:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.749077:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.749084:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.749187:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.749196:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.749208:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.749211:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.749214:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.749225:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.749228:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.749230:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.749234:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.749242:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.749251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.749253:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.749257:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.749260:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22929, 22929] 00000080:00200000:1.0:1775764895.749273:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.749276:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.749308:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93921280 i_blocks 90112 00000080:00200000:1.0:1775764895.749312:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.749328:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93921280, count: 4096 00000080:00200000:1.0:1775764895.749444:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93921280, bytes: 4096 00000080:00200000:1.0:1775764895.749447:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.749451:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22931, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.749453:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93921280, 93925376) 0 0 00000080:00200000:1.0:1775764895.749456:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.749462:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22930, 22930] 00020000:00200000:1.0:1775764895.749465:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93921280 chunk: [93921280, 93925376) 93925376, 4096 00020000:00200000:1.0:1775764895.749468:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.749480:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22930, 22930] 00000008:00010000:1.0:1775764895.749486:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.749489:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.749498:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.749507:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93921280 93925375) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.749516:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.749601:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.749609:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.749619:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93921280, 93925376) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.749626:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22930 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.749647:0:18302:0:(rw26.c:858:ll_write_end()) pos 93921280, len 4096, copied 4096 00000080:00200000:1.0:1775764895.749652:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000060056d82[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.749655:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000024dd64c3 17ffffc0001001 3:0 ffff90c20b3c90e0 22930 lru 00000080:00200000:1.0:1775764895.749661:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000018f969a4 22930: 1< 0 - - > 2< 93921280 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6717440 - | + - + - > 5< + - + - | 0 - | 2450 - -> 00000080:00200000:1.0:1775764895.749663:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000060056d82 00000080:00200000:1.0:1775764895.749664:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.749667:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.749676:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.749683:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.749690:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.749697:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.749709:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.749712:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.749714:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.749727:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.749729:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.749732:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.749737:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.749745:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.749753:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.749755:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.749760:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.749762:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22930, 22930] 00000080:00200000:1.0:1775764895.749774:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.749777:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.749810:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93925376 i_blocks 90112 00000080:00200000:1.0:1775764895.749814:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.749832:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93925376, count: 4096 00000080:00200000:1.0:1775764895.749844:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93925376, bytes: 4096 00000080:00200000:1.0:1775764895.749847:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.749851:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22932, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.749852:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93925376, 93929472) 0 0 00000080:00200000:1.0:1775764895.749854:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.749860:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22931, 22931] 00020000:00200000:1.0:1775764895.749863:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93925376 chunk: [93925376, 93929472) 93929472, 4096 00020000:00200000:1.0:1775764895.749866:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.749948:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22931, 22931] 00000008:00010000:1.0:1775764895.749956:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.749958:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.750031:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.750043:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93925376 93929471) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.750053:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.750061:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.750068:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.750076:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93925376, 93929472) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.750082:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22931 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.750115:0:18302:0:(rw26.c:858:ll_write_end()) pos 93925376, len 4096, copied 4096 00000080:00200000:1.0:1775764895.750132:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000011817bf3[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.750135:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000703a5a83 17ffffc0001001 3:0 ffff90c20b3ca7e0 22931 lru 00000080:00200000:1.0:1775764895.750140:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d16c4810 22931: 1< 0 - - > 2< 93925376 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6713344 - | + - + - > 5< + - + - | 0 - | 2451 - -> 00000080:00200000:1.0:1775764895.750143:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000011817bf3 00000080:00200000:1.0:1775764895.750144:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.750146:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.750155:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.750164:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.750172:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.750185:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.750197:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.750200:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.750203:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.750214:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.750217:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.750219:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.750223:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.750231:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.750241:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.750244:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.750248:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.750251:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22931, 22931] 00000080:00200000:1.0:1775764895.750263:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.750266:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.750300:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93929472 i_blocks 90112 00000080:00200000:1.0:1775764895.750303:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.750321:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93929472, count: 4096 00000080:00200000:1.0:1775764895.750332:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93929472, bytes: 4096 00000080:00200000:1.0:1775764895.750335:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.750339:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22933, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.750341:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93929472, 93933568) 0 0 00000080:00200000:1.0:1775764895.750343:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.750349:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22932, 22932] 00020000:00200000:1.0:1775764895.750352:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93929472 chunk: [93929472, 93933568) 93933568, 4096 00020000:00200000:1.0:1775764895.750355:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.750367:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22932, 22932] 00000008:00010000:1.0:1775764895.750372:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.750374:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.750382:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.750391:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93929472 93933567) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.750401:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.750409:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.750416:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.750425:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93929472, 93933568) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.750430:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22932 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.750449:0:18302:0:(rw26.c:858:ll_write_end()) pos 93929472, len 4096, copied 4096 00000080:00200000:1.0:1775764895.750454:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b3874f69[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.750457:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a5388b4e 17ffffc0001001 3:0 ffff90c20b3ca9a0 22932 lru 00000080:00200000:1.0:1775764895.750463:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000788a134e 22932: 1< 0 - - > 2< 93929472 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6709248 - | + - + - > 5< + - + - | 0 - | 2452 - -> 00000080:00200000:1.0:1775764895.750465:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b3874f69 00000080:00200000:1.0:1775764895.750466:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.750468:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.750476:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.750484:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.750492:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.750500:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.750511:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.750514:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.750517:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.751288:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.751292:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.751295:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.751299:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751307:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.751314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.751316:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.751321:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.751325:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22932, 22932] 00000080:00200000:1.0:1775764895.751341:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.751345:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.751395:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93933568 i_blocks 90112 00000080:00200000:1.0:1775764895.751400:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.751425:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93933568, count: 4096 00000080:00200000:1.0:1775764895.751435:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93933568, bytes: 4096 00000080:00200000:1.0:1775764895.751438:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.751441:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22934, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.751442:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93933568, 93937664) 0 0 00000080:00200000:1.0:1775764895.751444:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.751452:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22933, 22933] 00020000:00200000:1.0:1775764895.751456:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93933568 chunk: [93933568, 93937664) 93937664, 4096 00020000:00200000:1.0:1775764895.751458:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.751467:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22933, 22933] 00000008:00010000:1.0:1775764895.751474:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.751476:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.751486:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751493:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93933568 93937663) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751499:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751504:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751512:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.751518:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93933568, 93937664) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.751537:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22933 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.751575:0:18302:0:(rw26.c:858:ll_write_end()) pos 93933568, len 4096, copied 4096 00000080:00200000:1.0:1775764895.751581:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005b254ca5[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.751583:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000097cc4d8a 17ffffc0001001 3:0 ffff90c20b3ca2a0 22933 lru 00000080:00200000:1.0:1775764895.751588:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f6c1f5a8 22933: 1< 0 - - > 2< 93933568 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6705152 - | + - + - > 5< + - + - | 0 - | 2453 - -> 00000080:00200000:1.0:1775764895.751589:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005b254ca5 00000080:00200000:1.0:1775764895.751590:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.751592:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.751602:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.751608:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.751613:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.751617:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.751627:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.751629:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.751631:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.751639:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.751641:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.751643:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.751645:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751651:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.751656:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.751658:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.751660:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.751662:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22933, 22933] 00000080:00200000:1.0:1775764895.751669:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.751671:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.751691:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93937664 i_blocks 90112 00000080:00200000:1.0:1775764895.751694:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.751705:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93937664, count: 4096 00000080:00200000:1.0:1775764895.751711:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93937664, bytes: 4096 00000080:00200000:1.0:1775764895.751713:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.751715:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22935, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.751716:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93937664, 93941760) 0 0 00000080:00200000:1.0:1775764895.751718:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.751722:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22934, 22934] 00020000:00200000:1.0:1775764895.751724:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93937664 chunk: [93937664, 93941760) 93941760, 4096 00020000:00200000:1.0:1775764895.751725:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.751742:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22934, 22934] 00000008:00010000:1.0:1775764895.751746:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.751748:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.751754:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751760:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93937664 93941759) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751765:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751770:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751773:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.751779:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93937664, 93941760) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.751783:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22934 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.751795:0:18302:0:(rw26.c:858:ll_write_end()) pos 93937664, len 4096, copied 4096 00000080:00200000:1.0:1775764895.751798:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a6fb0c18[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.751800:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000326caa0b 17ffffc0001001 3:0 ffff90c20b3cad20 22934 lru 00000080:00200000:1.0:1775764895.751804:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006d52ef86 22934: 1< 0 - - > 2< 93937664 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6701056 - | + - + - > 5< + - + - | 0 - | 2454 - -> 00000080:00200000:1.0:1775764895.751805:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a6fb0c18 00000080:00200000:1.0:1775764895.751806:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.751807:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.751812:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.751817:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.751822:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.751827:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.751833:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.751835:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.751836:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.751843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.751844:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.751846:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.751848:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.751853:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.751858:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.751859:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.751861:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.751863:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22934, 22934] 00000080:00200000:1.0:1775764895.751897:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.751900:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.751920:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93941760 i_blocks 90112 00000080:00200000:1.0:1775764895.751922:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.751933:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93941760, count: 4096 00000080:00200000:1.0:1775764895.751939:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93941760, bytes: 4096 00000080:00200000:1.0:1775764895.751941:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.751944:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22936, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.751945:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93941760, 93945856) 0 0 00000080:00200000:1.0:1775764895.751946:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.751950:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22935, 22935] 00020000:00200000:1.0:1775764895.751952:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93941760 chunk: [93941760, 93945856) 93945856, 4096 00020000:00200000:1.0:1775764895.751954:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.751960:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22935, 22935] 00000008:00010000:1.0:1775764895.752078:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.752082:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.752090:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752100:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93941760 93945855) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752108:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752116:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752133:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.752142:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93941760, 93945856) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.752148:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22935 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.752169:0:18302:0:(rw26.c:858:ll_write_end()) pos 93941760, len 4096, copied 4096 00000080:00200000:1.0:1775764895.752173:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000069eefc71[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.752177:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000021da4491 17ffffc0001001 3:0 ffff90c20b3ca380 22935 lru 00000080:00200000:1.0:1775764895.752182:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000057c7ec70 22935: 1< 0 - - > 2< 93941760 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6696960 - | + - + - > 5< + - + - | 0 - | 2455 - -> 00000080:00200000:1.0:1775764895.752185:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000069eefc71 00000080:00200000:1.0:1775764895.752186:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.752188:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.752195:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752201:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752205:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752210:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.752218:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.752220:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.752222:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.752230:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.752232:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.752234:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.752238:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752246:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.752255:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.752257:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.752261:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.752264:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22935, 22935] 00000080:00200000:1.0:1775764895.752275:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.752278:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.752306:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93945856 i_blocks 90112 00000080:00200000:1.0:1775764895.752309:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.752321:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93945856, count: 4096 00000080:00200000:1.0:1775764895.752327:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93945856, bytes: 4096 00000080:00200000:1.0:1775764895.752329:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.752333:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22937, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.752335:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93945856, 93949952) 0 0 00000080:00200000:1.0:1775764895.752337:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.752343:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22936, 22936] 00020000:00200000:1.0:1775764895.752346:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93945856 chunk: [93945856, 93949952) 93949952, 4096 00020000:00200000:1.0:1775764895.752348:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.752360:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22936, 22936] 00000008:00010000:1.0:1775764895.752365:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.752367:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.752375:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752384:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93945856 93949951) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752392:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752406:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752413:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.752419:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93945856, 93949952) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.752423:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22936 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.752436:0:18302:0:(rw26.c:858:ll_write_end()) pos 93945856, len 4096, copied 4096 00000080:00200000:1.0:1775764895.752439:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c96b22cb[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.752441:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004519288f 17ffffc0001001 3:0 ffff90c20b3ca620 22936 lru 00000080:00200000:1.0:1775764895.752444:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000dde8639 22936: 1< 0 - - > 2< 93945856 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6692864 - | + - + - > 5< + - + - | 0 - | 2456 - -> 00000080:00200000:1.0:1775764895.752446:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c96b22cb 00000080:00200000:1.0:1775764895.752447:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.752448:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.752453:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752458:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752462:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752467:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.752477:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.752479:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.752480:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.752487:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.752489:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.752491:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.752493:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752498:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.752504:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.752505:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.752507:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.752509:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22936, 22936] 00000080:00200000:1.0:1775764895.752516:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.752518:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.752545:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93949952 i_blocks 90112 00000080:00200000:1.0:1775764895.752548:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.752563:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93949952, count: 4096 00000080:00200000:1.0:1775764895.752573:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93949952, bytes: 4096 00000080:00200000:1.0:1775764895.752577:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.752580:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22938, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.752581:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93949952, 93954048) 0 0 00000080:00200000:1.0:1775764895.752583:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.752587:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22937, 22937] 00020000:00200000:1.0:1775764895.752590:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93949952 chunk: [93949952, 93954048) 93954048, 4096 00020000:00200000:1.0:1775764895.752591:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.752598:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22937, 22937] 00000008:00010000:1.0:1775764895.752602:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.752604:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.752609:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752615:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93949952 93954047) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752621:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752625:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752629:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.752634:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93949952, 93954048) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.752638:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22937 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.752650:0:18302:0:(rw26.c:858:ll_write_end()) pos 93949952, len 4096, copied 4096 00000080:00200000:1.0:1775764895.752652:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000bd8d90e4[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.752654:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e4106106 17ffffc0001001 3:0 ffff90c20b3cae00 22937 lru 00000080:00200000:1.0:1775764895.752658:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a34f0aab 22937: 1< 0 - - > 2< 93949952 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6688768 - | + - + - > 5< + - + - | 0 - | 2457 - -> 00000080:00200000:1.0:1775764895.752660:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000bd8d90e4 00000080:00200000:1.0:1775764895.752661:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.752662:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.752667:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752672:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752677:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752681:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.752687:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.752689:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.752690:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.752697:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.752698:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.752700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.752702:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752706:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.752712:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.752713:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.752716:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.752718:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22937, 22937] 00000080:00200000:1.0:1775764895.752725:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.752727:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.752745:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93954048 i_blocks 90112 00000080:00200000:1.0:1775764895.752748:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.752758:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93954048, count: 4096 00000080:00200000:1.0:1775764895.752764:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93954048, bytes: 4096 00000080:00200000:1.0:1775764895.752766:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.752768:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22939, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.752769:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93954048, 93958144) 0 0 00000080:00200000:1.0:1775764895.752771:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.752774:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22938, 22938] 00020000:00200000:1.0:1775764895.752776:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93954048 chunk: [93954048, 93958144) 93958144, 4096 00020000:00200000:1.0:1775764895.752777:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.752783:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22938, 22938] 00000008:00010000:1.0:1775764895.752786:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.752788:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.752793:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752799:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93954048 93958143) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752804:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752808:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752812:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.752817:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93954048, 93958144) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.752820:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22938 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.752833:0:18302:0:(rw26.c:858:ll_write_end()) pos 93954048, len 4096, copied 4096 00000080:00200000:1.0:1775764895.752836:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000029f5cafb[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.752838:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b0169f9b 17ffffc0001001 3:0 ffff90c20b3ca540 22938 lru 00000080:00200000:1.0:1775764895.752841:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001e88514a 22938: 1< 0 - - > 2< 93954048 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6684672 - | + - + - > 5< + - + - | 0 - | 2458 - -> 00000080:00200000:1.0:1775764895.752843:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000029f5cafb 00000080:00200000:1.0:1775764895.752844:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.752845:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.752850:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752855:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752859:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.752863:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.752888:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.752890:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.752891:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.752898:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.752899:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.752901:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.752903:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.752909:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.752914:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.752915:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.752917:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.752919:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22938, 22938] 00000080:00200000:1.0:1775764895.752926:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.752928:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.752946:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93958144 i_blocks 90112 00000080:00200000:1.0:1775764895.752949:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.752959:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93958144, count: 4096 00000080:00200000:1.0:1775764895.753261:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93958144, bytes: 4096 00000080:00200000:1.0:1775764895.753264:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.753266:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22940, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.753268:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93958144, 93962240) 0 0 00000080:00200000:1.0:1775764895.753269:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.753274:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22939, 22939] 00020000:00200000:1.0:1775764895.753276:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93958144 chunk: [93958144, 93962240) 93962240, 4096 00020000:00200000:1.0:1775764895.753278:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.753286:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22939, 22939] 00000008:00010000:1.0:1775764895.753289:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.753291:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.753298:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.753304:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93958144 93962239) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.753310:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.753315:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.753318:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.753323:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93958144, 93962240) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.753326:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22939 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.753342:0:18302:0:(rw26.c:858:ll_write_end()) pos 93958144, len 4096, copied 4096 00000080:00200000:1.0:1775764895.753346:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000629b6ba1[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.753348:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000018e62233 17ffffc0001001 3:0 ffff90c20b3ca460 22939 lru 00000080:00200000:1.0:1775764895.753353:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ecfef811 22939: 1< 0 - - > 2< 93958144 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6680576 - | + - + - > 5< + - + - | 0 - | 2459 - -> 00000080:00200000:1.0:1775764895.753355:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000629b6ba1 00000080:00200000:1.0:1775764895.753356:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.753359:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.753368:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.753375:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.753382:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.753396:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.753409:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.753412:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.753414:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.753423:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.753425:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.753427:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.753429:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.753435:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.753440:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.753441:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.753444:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.753446:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22939, 22939] 00000080:00200000:1.0:1775764895.753453:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.753455:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.753475:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93962240 i_blocks 90112 00000080:00200000:1.0:1775764895.753478:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.753489:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93962240, count: 4096 00000080:00200000:1.0:1775764895.753495:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93962240, bytes: 4096 00000080:00200000:1.0:1775764895.753497:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.753500:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22941, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.753501:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93962240, 93966336) 0 0 00000080:00200000:1.0:1775764895.753502:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.753506:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22940, 22940] 00020000:00200000:1.0:1775764895.753508:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93962240 chunk: [93962240, 93966336) 93966336, 4096 00020000:00200000:1.0:1775764895.753509:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.753516:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22940, 22940] 00000008:00010000:1.0:1775764895.753712:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.753716:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.753725:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.753817:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93962240 93966335) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.753825:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.753831:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.753838:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.753847:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93962240, 93966336) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.753853:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22940 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.753899:0:18302:0:(rw26.c:858:ll_write_end()) pos 93962240, len 4096, copied 4096 00000080:00200000:1.0:1775764895.753903:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000012cffc84[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.753906:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000180c9408 17ffffc0001001 3:0 ffff90c20b3caa80 22940 lru 00000080:00200000:1.0:1775764895.753913:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004f5235b8 22940: 1< 0 - - > 2< 93962240 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6676480 - | + - + - > 5< + - + - | 0 - | 2460 - -> 00000080:00200000:1.0:1775764895.753915:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000012cffc84 00000080:00200000:1.0:1775764895.753916:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.753919:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.753929:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.753936:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.753942:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.753950:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.754022:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.754026:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.754029:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.754041:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.754044:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.754046:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.754050:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.754058:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.754066:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.754069:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.754073:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.754076:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22940, 22940] 00000080:00200000:1.0:1775764895.754088:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.754092:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.754134:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93966336 i_blocks 90112 00000080:00200000:1.0:1775764895.754138:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.754157:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93966336, count: 4096 00000080:00200000:1.0:1775764895.754169:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93966336, bytes: 4096 00000080:00200000:1.0:1775764895.754171:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.754175:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22942, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.754177:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93966336, 93970432) 0 0 00000080:00200000:1.0:1775764895.754179:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.754185:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22941, 22941] 00020000:00200000:1.0:1775764895.754188:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93966336 chunk: [93966336, 93970432) 93970432, 4096 00020000:00200000:1.0:1775764895.754190:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.754200:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22941, 22941] 00000008:00010000:1.0:1775764895.754205:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.754208:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.754216:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.754224:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93966336 93970431) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.754231:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.754238:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.754250:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.754257:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93966336, 93970432) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.754262:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22941 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.754279:0:18302:0:(rw26.c:858:ll_write_end()) pos 93966336, len 4096, copied 4096 00000080:00200000:1.0:1775764895.754283:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f7c9c0d5[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.754285:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a501bfca 17ffffc0001001 3:0 ffff90c20b3ca700 22941 lru 00000080:00200000:1.0:1775764895.754291:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000090a03de 22941: 1< 0 - - > 2< 93966336 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6672384 - | + - + - > 5< + - + - | 0 - | 2461 - -> 00000080:00200000:1.0:1775764895.754293:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f7c9c0d5 00000080:00200000:1.0:1775764895.754294:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.754297:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.754305:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.754312:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.754320:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.754327:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.754338:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.754341:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.754344:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.754354:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.754358:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.754360:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.754364:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.754372:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.754381:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.754383:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.754387:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.754390:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22941, 22941] 00000080:00200000:1.0:1775764895.754401:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.754404:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.754437:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93970432 i_blocks 90112 00000080:00200000:1.0:1775764895.754441:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.754456:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93970432, count: 4096 00000080:00200000:1.0:1775764895.754467:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93970432, bytes: 4096 00000080:00200000:1.0:1775764895.754470:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.754474:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22943, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.754475:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93970432, 93974528) 0 0 00000080:00200000:1.0:1775764895.754478:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.754484:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22942, 22942] 00020000:00200000:1.0:1775764895.754487:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93970432 chunk: [93970432, 93974528) 93974528, 4096 00020000:00200000:1.0:1775764895.754489:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.754501:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22942, 22942] 00000008:00010000:1.0:1775764895.754506:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.754508:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.754516:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.755115:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93970432 93974527) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.755140:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.755147:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.755153:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.755164:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93970432, 93974528) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.755171:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22942 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.755208:0:18302:0:(rw26.c:858:ll_write_end()) pos 93970432, len 4096, copied 4096 00000080:00200000:1.0:1775764895.755213:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fb4fdfad[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.755216:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c49afd2e 17ffffc0001021 2:0 ffff90c20b3cac40 22942 lru 00000080:00200000:1.0:1775764895.755222:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000090e0454c 22942: 1< 0 - - > 2< 93970432 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6668288 - | + - + - > 5< + - + - | 0 - | 2462 - -> 00000080:00200000:1.0:1775764895.755225:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fb4fdfad 00000080:00200000:1.0:1775764895.755226:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.755230:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.755242:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.755251:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.755259:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.755267:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.755281:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.755285:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.755288:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.755303:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.755307:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.755310:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.755314:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.755324:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.755335:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.755338:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.755344:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.755348:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22942, 22942] 00000080:00200000:1.0:1775764895.755365:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.755368:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.755408:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93974528 i_blocks 90112 00000080:00200000:1.0:1775764895.755412:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.755432:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93974528, count: 4096 00000080:00200000:1.0:1775764895.755445:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93974528, bytes: 4096 00000080:00200000:1.0:1775764895.755449:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.755467:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22944, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.755468:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93974528, 93978624) 0 0 00000080:00200000:1.0:1775764895.755471:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.755479:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22943, 22943] 00020000:00200000:1.0:1775764895.755483:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93974528 chunk: [93974528, 93978624) 93978624, 4096 00020000:00200000:1.0:1775764895.755487:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.755503:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22943, 22943] 00000008:00010000:1.0:1775764895.755512:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.755517:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.756130:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.756144:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93974528 93978623) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.756154:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.756162:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.756168:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.756179:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93974528, 93978624) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.756188:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22943 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.756214:0:18302:0:(rw26.c:858:ll_write_end()) pos 93974528, len 4096, copied 4096 00000080:00200000:1.0:1775764895.756221:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003996825c[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.756224:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001a7604b9 17ffffc0001001 3:0 ffff90c20b3ca8c0 22943 lru 00000080:00200000:1.0:1775764895.756230:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000091f5216d 22943: 1< 0 - - > 2< 93974528 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6664192 - | + - + - > 5< + - + - | 0 - | 2463 - -> 00000080:00200000:1.0:1775764895.756232:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003996825c 00000080:00200000:1.0:1775764895.756233:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.756236:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.756249:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.756257:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.756265:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.756274:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.756288:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.756291:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.756293:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.756308:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.756311:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.756314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.756318:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.756326:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.756337:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.756339:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.756344:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.756347:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22943, 22943] 00000080:00200000:1.0:1775764895.756363:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.756367:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.756412:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93978624 i_blocks 90112 00000080:00200000:1.0:1775764895.756417:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.756440:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93978624, count: 4096 00000080:00200000:1.0:1775764895.756453:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93978624, bytes: 4096 00000080:00200000:1.0:1775764895.756456:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.756460:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22945, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.756462:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93978624, 93982720) 0 0 00000080:00200000:1.0:1775764895.756465:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.756473:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22944, 22944] 00020000:00200000:1.0:1775764895.756476:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93978624 chunk: [93978624, 93982720) 93982720, 4096 00020000:00200000:1.0:1775764895.756479:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.756491:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22944, 22944] 00000008:00010000:1.0:1775764895.756498:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.756501:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.756509:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.756518:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93978624 93982719) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.762998:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.763018:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.763023:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.763043:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93978624, 93982720) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.763064:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22944 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.763126:0:18302:0:(rw26.c:858:ll_write_end()) pos 93978624, len 4096, copied 4096 00000080:00200000:1.0:1775764895.763135:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002f02b5e9[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.763139:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c9dc67fa 17ffffc0001001 3:0 ffff90c20b3cab60 22944 lru 00000080:00200000:1.0:1775764895.763146:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e80f2c50 22944: 1< 0 - - > 2< 93978624 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6660096 - | + - + - > 5< + - + - | 0 - | 2464 - -> 00000080:00200000:1.0:1775764895.763149:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002f02b5e9 00000080:00200000:1.0:1775764895.763150:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.763155:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.763177:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.763187:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.763194:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.763201:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.763225:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.763230:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.763232:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.763253:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.763256:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.763259:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.763262:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.763269:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.763279:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.763281:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.763287:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.763293:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22944, 22944] 00000080:00200000:1.0:1775764895.763320:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.763323:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.763373:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93982720 i_blocks 90112 00000080:00200000:1.0:1775764895.763377:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.763411:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93982720, count: 4096 00000080:00200000:1.0:1775764895.763427:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93982720, bytes: 4096 00000080:00200000:1.0:1775764895.763431:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.763437:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22946, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.763439:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93982720, 93986816) 0 0 00000080:00200000:1.0:1775764895.763443:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.763456:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22945, 22945] 00020000:00200000:1.0:1775764895.763462:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93982720 chunk: [93982720, 93986816) 93986816, 4096 00020000:00200000:1.0:1775764895.763465:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.763479:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22945, 22945] 00000008:00010000:1.0:1775764895.763487:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.763491:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.763507:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.763516:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93982720 93986815) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.763523:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.763531:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.763538:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.765081:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93982720, 93986816) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.765101:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22945 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.765162:0:18302:0:(rw26.c:858:ll_write_end()) pos 93982720, len 4096, copied 4096 00000080:00200000:1.0:1775764895.765171:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000ac74d61[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.765174:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008604455f 17ffffc0001001 3:0 ffff90c20b3ca000 22945 lru 00000080:00200000:1.0:1775764895.765182:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d1653ef8 22945: 1< 0 - - > 2< 93982720 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6656000 - | + - + - > 5< + - + - | 0 - | 2465 - -> 00000080:00200000:1.0:1775764895.765184:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000ac74d61 00000080:00200000:1.0:1775764895.765185:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.765190:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.765213:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.765224:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.765232:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.765242:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.765264:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.765269:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.765272:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.765297:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.765301:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.765304:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.765309:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.765319:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.765331:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.765334:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.765342:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.765348:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22945, 22945] 00000080:00200000:1.0:1775764895.765377:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.765381:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.765438:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93986816 i_blocks 90112 00000080:00200000:1.0:1775764895.765443:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.765473:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93986816, count: 4096 00000080:00200000:1.0:1775764895.765483:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93986816, bytes: 4096 00000080:00200000:1.0:1775764895.765487:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.765490:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22947, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.765492:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93986816, 93990912) 0 0 00000080:00200000:1.0:1775764895.765495:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.765506:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22946, 22946] 00020000:00200000:1.0:1775764895.765512:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93986816 chunk: [93986816, 93990912) 93990912, 4096 00020000:00200000:1.0:1775764895.765515:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.765532:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22946, 22946] 00000008:00010000:1.0:1775764895.765543:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.765548:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.765567:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.765585:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93986816 93990911) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.765597:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.765605:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.765613:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.765622:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93986816, 93990912) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.765629:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22946 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.765650:0:18302:0:(rw26.c:858:ll_write_end()) pos 93986816, len 4096, copied 4096 00000080:00200000:1.0:1775764895.765655:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d8610366[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.765658:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000704d4d6a 17ffffc0001001 3:0 ffff90c20b3caee0 22946 lru 00000080:00200000:1.0:1775764895.765662:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f353debc 22946: 1< 0 - - > 2< 93986816 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6651904 - | + - + - > 5< + - + - | 0 - | 2466 - -> 00000080:00200000:1.0:1775764895.765663:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d8610366 00000080:00200000:1.0:1775764895.765664:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.765666:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.765673:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.765678:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.765682:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.765687:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.765694:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.765696:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.765698:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.765705:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.765707:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.765708:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.765711:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.765717:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.765722:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.765724:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.765726:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.766920:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22946, 22946] 00000080:00200000:1.0:1775764895.766945:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.766949:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.767004:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93990912 i_blocks 90112 00000080:00200000:1.0:1775764895.767008:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.767033:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93990912, count: 4096 00000080:00200000:1.0:1775764895.767043:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93990912, bytes: 4096 00000080:00200000:1.0:1775764895.767047:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.767052:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22948, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.767054:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93990912, 93995008) 0 0 00000080:00200000:1.0:1775764895.767056:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.767066:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22947, 22947] 00020000:00200000:1.0:1775764895.767071:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93990912 chunk: [93990912, 93995008) 93995008, 4096 00020000:00200000:1.0:1775764895.767073:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.767090:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22947, 22947] 00000008:00010000:1.0:1775764895.767102:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.767106:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.767136:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767151:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93990912 93995007) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767162:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767172:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767179:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.767188:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93990912, 93995008) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.767195:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22947 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.767227:0:18302:0:(rw26.c:858:ll_write_end()) pos 93990912, len 4096, copied 4096 00000080:00200000:1.0:1775764895.767231:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000370b2ead[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.767234:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cfb20f35 17ffffc0001001 3:0 ffff90c20b3ca0e0 22947 lru 00000080:00200000:1.0:1775764895.767238:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000052a10392 22947: 1< 0 - - > 2< 93990912 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6647808 - | + - + - > 5< + - + - | 0 - | 2467 - -> 00000080:00200000:1.0:1775764895.767240:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000370b2ead 00000080:00200000:1.0:1775764895.767240:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.767243:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.767255:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.767261:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.767266:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.767275:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.767287:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.767289:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.767291:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.767305:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.767307:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.767309:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.767312:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767318:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.767326:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.767327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.767332:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.767335:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22947, 22947] 00000080:00200000:1.0:1775764895.767345:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.767348:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.767377:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93995008 i_blocks 90112 00000080:00200000:1.0:1775764895.767381:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.767397:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93995008, count: 4096 00000080:00200000:1.0:1775764895.767408:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93995008, bytes: 4096 00000080:00200000:1.0:1775764895.767411:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.767415:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22949, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.767417:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93995008, 93999104) 0 0 00000080:00200000:1.0:1775764895.767419:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.767424:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22948, 22948] 00020000:00200000:1.0:1775764895.767426:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93995008 chunk: [93995008, 93999104) 93999104, 4096 00020000:00200000:1.0:1775764895.767427:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.767436:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22948, 22948] 00000008:00010000:1.0:1775764895.767439:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.767441:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.767447:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767454:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93995008 93999103) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767459:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767464:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767467:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.767472:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93995008, 93999104) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.767475:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22948 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.767489:0:18302:0:(rw26.c:858:ll_write_end()) pos 93995008, len 4096, copied 4096 00000080:00200000:1.0:1775764895.767492:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000da264086[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.767494:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000000ba2884 17ffffc0001001 3:0 ffff90c20b3ca1c0 22948 lru 00000080:00200000:1.0:1775764895.767498:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009e094f23 22948: 1< 0 - - > 2< 93995008 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6643712 - | + - + - > 5< + - + - | 0 - | 2468 - -> 00000080:00200000:1.0:1775764895.767499:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000da264086 00000080:00200000:1.0:1775764895.767500:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.767502:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.767507:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.767512:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.767517:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.767721:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.767800:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.767804:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.767806:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.767815:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.767817:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.767818:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.767821:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767827:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.767832:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.767834:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.767838:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.767840:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22948, 22948] 00000080:00200000:1.0:1775764895.767848:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.767851:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.767895:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 93999104 i_blocks 90112 00000080:00200000:1.0:1775764895.767898:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.767912:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 93999104, count: 4096 00000080:00200000:1.0:1775764895.767919:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 93999104, bytes: 4096 00000080:00200000:1.0:1775764895.767921:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.767924:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22950, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.767925:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [93999104, 94003200) 0 0 00000080:00200000:1.0:1775764895.767927:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.767931:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22949, 22949] 00020000:00200000:1.0:1775764895.767933:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 93999104 chunk: [93999104, 94003200) 94003200, 4096 00020000:00200000:1.0:1775764895.767935:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.767942:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22949, 22949] 00000008:00010000:1.0:1775764895.767946:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.767949:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.767954:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767962:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (93999104 94003199) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767978:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767983:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.767992:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.767996:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [93999104, 94003200) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.768000:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22949 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.768027:0:18302:0:(rw26.c:858:ll_write_end()) pos 93999104, len 4096, copied 4096 00000080:00200000:1.0:1775764895.768030:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f19df991[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.768032:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000017963139 17ffffc0001001 3:0 ffff90c20b3e68c0 22949 lru 00000080:00200000:1.0:1775764895.768036:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002a9e64b2 22949: 1< 0 - - > 2< 93999104 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6639616 - | + - + - > 5< + - + - | 0 - | 2469 - -> 00000080:00200000:1.0:1775764895.768038:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f19df991 00000080:00200000:1.0:1775764895.768039:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.768040:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.768046:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.768051:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.768055:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.768060:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.768067:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.768068:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.768070:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.768077:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.768079:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.768081:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.768084:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768091:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.768100:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.768102:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.768106:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.768109:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22949, 22949] 00000080:00200000:1.0:1775764895.768130:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.768133:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.768162:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94003200 i_blocks 90112 00000080:00200000:1.0:1775764895.768166:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.768182:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94003200, count: 4096 00000080:00200000:1.0:1775764895.768190:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94003200, bytes: 4096 00000080:00200000:1.0:1775764895.768192:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.768194:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22951, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.768195:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94003200, 94007296) 0 0 00000080:00200000:1.0:1775764895.768197:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.768201:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22950, 22950] 00020000:00200000:1.0:1775764895.768203:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94003200 chunk: [94003200, 94007296) 94007296, 4096 00020000:00200000:1.0:1775764895.768205:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.768212:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22950, 22950] 00000008:00010000:1.0:1775764895.768215:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.768217:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.768223:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768229:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94003200 94007295) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768235:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768240:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768243:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.768248:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94003200, 94007296) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.768251:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22950 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.768263:0:18302:0:(rw26.c:858:ll_write_end()) pos 94003200, len 4096, copied 4096 00000080:00200000:1.0:1775764895.768266:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005ebcae3f[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.768268:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c3ad3523 17ffffc0001001 3:0 ffff90c20b3e6b60 22950 lru 00000080:00200000:1.0:1775764895.768272:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000514d16b5 22950: 1< 0 - - > 2< 94003200 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6635520 - | + - + - > 5< + - + - | 0 - | 2470 - -> 00000080:00200000:1.0:1775764895.768273:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005ebcae3f 00000080:00200000:1.0:1775764895.768274:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.768275:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.768280:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.768285:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.768290:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.768295:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.768302:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.768303:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.768305:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.768311:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.768312:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.768314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.768316:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768321:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.768326:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.768327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.768330:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.768333:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22950, 22950] 00000080:00200000:1.0:1775764895.768342:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.768346:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.768373:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94007296 i_blocks 90112 00000080:00200000:1.0:1775764895.768376:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.768393:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94007296, count: 4096 00000080:00200000:1.0:1775764895.768403:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94007296, bytes: 4096 00000080:00200000:1.0:1775764895.768406:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.768409:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22952, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.768411:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94007296, 94011392) 0 0 00000080:00200000:1.0:1775764895.768414:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.768419:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22951, 22951] 00020000:00200000:1.0:1775764895.768422:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94007296 chunk: [94007296, 94011392) 94011392, 4096 00020000:00200000:1.0:1775764895.768424:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.768433:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22951, 22951] 00000008:00010000:1.0:1775764895.768436:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.768438:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.768444:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768450:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94007296 94011391) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768456:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768461:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.768464:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.768469:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94007296, 94011392) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.768472:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22951 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.768484:0:18302:0:(rw26.c:858:ll_write_end()) pos 94007296, len 4096, copied 4096 00000080:00200000:1.0:1775764895.768487:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000075a73d66[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.768489:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000055862b53 17ffffc0001001 3:0 ffff90c20b3e6000 22951 lru 00000080:00200000:1.0:1775764895.768493:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000057fcef5 22951: 1< 0 - - > 2< 94007296 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6631424 - | + - + - > 5< + - + - | 0 - | 2471 - -> 00000080:00200000:1.0:1775764895.768495:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000075a73d66 00000080:00200000:1.0:1775764895.768496:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.768497:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.768502:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.768507:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.768512:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.768516:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.769043:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.769046:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.769048:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.769059:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.769060:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.769062:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.769065:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769071:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.769078:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.769080:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.769083:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.769086:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22951, 22951] 00000080:00200000:1.0:1775764895.769101:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.769104:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.769153:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94011392 i_blocks 90112 00000080:00200000:1.0:1775764895.769158:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.769178:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94011392, count: 4096 00000080:00200000:1.0:1775764895.769188:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94011392, bytes: 4096 00000080:00200000:1.0:1775764895.769190:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.769193:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22953, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.769194:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94011392, 94015488) 0 0 00000080:00200000:1.0:1775764895.769196:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.769201:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22952, 22952] 00020000:00200000:1.0:1775764895.769204:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94011392 chunk: [94011392, 94015488) 94015488, 4096 00020000:00200000:1.0:1775764895.769206:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.769215:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22952, 22952] 00000008:00010000:1.0:1775764895.769219:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.769221:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.769229:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769236:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94011392 94015487) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769241:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769250:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769255:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.769259:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94011392, 94015488) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.769264:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22952 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.769285:0:18302:0:(rw26.c:858:ll_write_end()) pos 94011392, len 4096, copied 4096 00000080:00200000:1.0:1775764895.769289:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007b08c1ec[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.769290:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b1f15fca 17ffffc0001001 3:0 ffff90c20b3e6ee0 22952 lru 00000080:00200000:1.0:1775764895.769294:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000061a1fa2d 22952: 1< 0 - - > 2< 94011392 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6627328 - | + - + - > 5< + - + - | 0 - | 2472 - -> 00000080:00200000:1.0:1775764895.769295:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007b08c1ec 00000080:00200000:1.0:1775764895.769296:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.769298:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.769305:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.769310:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.769314:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.769318:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.769329:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.769332:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.769334:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.769344:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.769346:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.769348:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.769352:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769359:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.769368:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.769370:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.769374:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.769377:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22952, 22952] 00000080:00200000:1.0:1775764895.769387:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.769390:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.769419:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94015488 i_blocks 90112 00000080:00200000:1.0:1775764895.769422:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.769434:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94015488, count: 4096 00000080:00200000:1.0:1775764895.769441:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94015488, bytes: 4096 00000080:00200000:1.0:1775764895.769442:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.769445:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22954, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.769446:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94015488, 94019584) 0 0 00000080:00200000:1.0:1775764895.769448:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.769452:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22953, 22953] 00020000:00200000:1.0:1775764895.769455:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94015488 chunk: [94015488, 94019584) 94019584, 4096 00020000:00200000:1.0:1775764895.769456:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.769463:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22953, 22953] 00000008:00010000:1.0:1775764895.769467:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.769469:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.769475:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769480:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94015488 94019583) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769486:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769490:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769494:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.769498:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94015488, 94019584) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.769502:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22953 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.769515:0:18302:0:(rw26.c:858:ll_write_end()) pos 94015488, len 4096, copied 4096 00000080:00200000:1.0:1775764895.769518:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009cbb52bb[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.769521:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ed1a7d4a 17ffffc0001001 3:0 ffff90c20b3e60e0 22953 lru 00000080:00200000:1.0:1775764895.769527:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000885d6c60 22953: 1< 0 - - > 2< 94015488 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6623232 - | + - + - > 5< + - + - | 0 - | 2473 - -> 00000080:00200000:1.0:1775764895.769529:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009cbb52bb 00000080:00200000:1.0:1775764895.769530:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.769532:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.769541:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.769549:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.769556:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.769564:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.769574:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.769577:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.769579:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.769588:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.769591:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.769593:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.769597:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769606:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.769614:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.769617:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.769621:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.769623:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22953, 22953] 00000080:00200000:1.0:1775764895.769634:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.769637:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.769665:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94019584 i_blocks 90112 00000080:00200000:1.0:1775764895.769669:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.769680:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94019584, count: 4096 00000080:00200000:1.0:1775764895.769687:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94019584, bytes: 4096 00000080:00200000:1.0:1775764895.769688:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.769691:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22955, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.769692:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94019584, 94023680) 0 0 00000080:00200000:1.0:1775764895.769694:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.769698:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22954, 22954] 00020000:00200000:1.0:1775764895.769700:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94019584 chunk: [94019584, 94023680) 94023680, 4096 00020000:00200000:1.0:1775764895.769701:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.769708:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22954, 22954] 00000008:00010000:1.0:1775764895.769711:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.769713:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.769719:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.769725:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94019584 94023679) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.770927:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.770936:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.770941:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.770954:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94019584, 94023680) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.771017:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22954 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.771055:0:18302:0:(rw26.c:858:ll_write_end()) pos 94019584, len 4096, copied 4096 00000080:00200000:1.0:1775764895.771061:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000055600e72[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.771063:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000054d67c0b 17ffffc0001001 3:0 ffff90c20b3e61c0 22954 lru 00000080:00200000:1.0:1775764895.771068:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000098a00c45 22954: 1< 0 - - > 2< 94019584 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6619136 - | + - + - > 5< + - + - | 0 - | 2474 - -> 00000080:00200000:1.0:1775764895.771069:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000055600e72 00000080:00200000:1.0:1775764895.771070:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.771072:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.771084:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.771094:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.771101:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.771108:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.771136:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.771140:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.771143:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.771166:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.771170:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.771173:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.771178:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.771185:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.771193:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.771195:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.771200:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.771205:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22954, 22954] 00000080:00200000:1.0:1775764895.771227:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.771230:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.771276:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94023680 i_blocks 90112 00000080:00200000:1.0:1775764895.771280:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.771303:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94023680, count: 4096 00000080:00200000:1.0:1775764895.771312:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94023680, bytes: 4096 00000080:00200000:1.0:1775764895.771315:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.771319:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22956, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.771320:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94023680, 94027776) 0 0 00000080:00200000:1.0:1775764895.771323:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.771332:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22955, 22955] 00020000:00200000:1.0:1775764895.771336:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94023680 chunk: [94023680, 94027776) 94027776, 4096 00020000:00200000:1.0:1775764895.771339:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.771354:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22955, 22955] 00000008:00010000:1.0:1775764895.771364:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.771368:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.771381:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.771393:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94023680 94027775) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.771402:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.771410:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.771417:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.771424:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94023680, 94027776) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.771429:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22955 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.771443:0:18302:0:(rw26.c:858:ll_write_end()) pos 94023680, len 4096, copied 4096 00000080:00200000:1.0:1775764895.771446:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d3e40166[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.771448:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d9de6b74 17ffffc0001001 3:0 ffff90c20b3e67e0 22955 lru 00000080:00200000:1.0:1775764895.771452:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e35af637 22955: 1< 0 - - > 2< 94023680 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6615040 - | + - + - > 5< + - + - | 0 - | 2475 - -> 00000080:00200000:1.0:1775764895.771453:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d3e40166 00000080:00200000:1.0:1775764895.771454:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.771455:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.771461:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.771466:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.771470:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.771479:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.771486:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.771488:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.771489:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.771496:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.771498:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.771500:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.771503:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.771508:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.771513:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.771514:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.771517:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.772703:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22955, 22955] 00000080:00200000:1.0:1775764895.772806:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.772810:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.772860:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94027776 i_blocks 90112 00000080:00200000:1.0:1775764895.772864:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.772904:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94027776, count: 4096 00000080:00200000:1.0:1775764895.772915:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94027776, bytes: 4096 00000080:00200000:1.0:1775764895.772919:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.772925:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22957, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.772927:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94027776, 94031872) 0 0 00000080:00200000:1.0:1775764895.772929:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.772939:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22956, 22956] 00020000:00200000:1.0:1775764895.772944:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94027776 chunk: [94027776, 94031872) 94031872, 4096 00020000:00200000:1.0:1775764895.772946:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.772961:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22956, 22956] 00000008:00010000:1.0:1775764895.772978:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.772981:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.772999:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773009:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94027776 94031871) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773017:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773022:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773026:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.773033:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94027776, 94031872) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.773041:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22956 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.773071:0:18302:0:(rw26.c:858:ll_write_end()) pos 94027776, len 4096, copied 4096 00000080:00200000:1.0:1775764895.773075:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ee180d4c[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.773077:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d775f869 17ffffc0001001 3:0 ffff90c20b3e69a0 22956 lru 00000080:00200000:1.0:1775764895.773082:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000029d2e9e5 22956: 1< 0 - - > 2< 94027776 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6610944 - | + - + - > 5< + - + - | 0 - | 2476 - -> 00000080:00200000:1.0:1775764895.773085:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ee180d4c 00000080:00200000:1.0:1775764895.773086:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.773090:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.773106:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.773115:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.773157:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.773166:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.773185:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.773190:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.773193:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.773215:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.773218:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.773220:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.773224:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773232:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.773243:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.773246:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.773251:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.773255:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22956, 22956] 00000080:00200000:1.0:1775764895.773265:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.773273:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.773304:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94031872 i_blocks 90112 00000080:00200000:1.0:1775764895.773308:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.773324:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94031872, count: 4096 00000080:00200000:1.0:1775764895.773334:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94031872, bytes: 4096 00000080:00200000:1.0:1775764895.773337:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.773340:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22958, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.773342:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94031872, 94035968) 0 0 00000080:00200000:1.0:1775764895.773344:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.773349:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22957, 22957] 00020000:00200000:1.0:1775764895.773352:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94031872 chunk: [94031872, 94035968) 94035968, 4096 00020000:00200000:1.0:1775764895.773355:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.773366:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22957, 22957] 00000008:00010000:1.0:1775764895.773372:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.773375:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.773383:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773393:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94031872 94035967) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773401:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773407:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773419:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.773427:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94031872, 94035968) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.773433:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22957 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.773457:0:18302:0:(rw26.c:858:ll_write_end()) pos 94031872, len 4096, copied 4096 00000080:00200000:1.0:1775764895.773462:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000047fac6bc[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.773465:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000948dbd0b 17ffffc0001021 2:0 ffff90c20b3e62a0 22957 lru 00000080:00200000:1.0:1775764895.773471:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001c0b1715 22957: 1< 0 - - > 2< 94031872 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6606848 - | + - + - > 5< + - + - | 0 - | 2477 - -> 00000080:00200000:1.0:1775764895.773473:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000047fac6bc 00000080:00200000:1.0:1775764895.773474:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.773476:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.773482:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.773487:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.773491:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.773495:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.773503:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.773504:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.773506:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.773514:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.773516:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.773517:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.773535:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773543:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.773552:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.773555:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.773559:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.773562:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22957, 22957] 00000080:00200000:1.0:1775764895.773573:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.773576:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.773604:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94035968 i_blocks 90112 00000080:00200000:1.0:1775764895.773608:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.773624:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94035968, count: 4096 00000080:00200000:1.0:1775764895.773635:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94035968, bytes: 4096 00000080:00200000:1.0:1775764895.773637:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.773641:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22959, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.773642:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94035968, 94040064) 0 0 00000080:00200000:1.0:1775764895.773645:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.773650:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22958, 22958] 00020000:00200000:1.0:1775764895.773653:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94035968 chunk: [94035968, 94040064) 94040064, 4096 00020000:00200000:1.0:1775764895.773655:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.773666:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22958, 22958] 00000008:00010000:1.0:1775764895.773670:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.773672:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.773678:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773684:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94035968 94040063) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773690:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773694:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773698:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.773703:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94035968, 94040064) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.773707:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22958 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.773720:0:18302:0:(rw26.c:858:ll_write_end()) pos 94035968, len 4096, copied 4096 00000080:00200000:1.0:1775764895.773723:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c31d6265[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.773725:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000096988a14 17ffffc0001001 3:0 ffff90c20b3e6d20 22958 lru 00000080:00200000:1.0:1775764895.773807:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004908a32e 22958: 1< 0 - - > 2< 94035968 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6602752 - | + - + - > 5< + - + - | 0 - | 2478 - -> 00000080:00200000:1.0:1775764895.773809:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c31d6265 00000080:00200000:1.0:1775764895.773810:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.773812:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.773818:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.773824:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.773829:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.773834:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.773841:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.773843:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.773844:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.773851:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.773853:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.773854:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.773857:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.773862:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.773867:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.773888:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.773891:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.773893:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22958, 22958] 00000080:00200000:1.0:1775764895.773900:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.773902:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.773921:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94040064 i_blocks 90112 00000080:00200000:1.0:1775764895.773924:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.773936:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94040064, count: 4096 00000080:00200000:1.0:1775764895.773943:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94040064, bytes: 4096 00000080:00200000:1.0:1775764895.773946:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.773950:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22960, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.773951:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94040064, 94044160) 0 0 00000080:00200000:1.0:1775764895.773954:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.773959:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22959, 22959] 00020000:00200000:1.0:1775764895.773962:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94040064 chunk: [94040064, 94044160) 94044160, 4096 00020000:00200000:1.0:1775764895.773964:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.773976:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22959, 22959] 00000008:00010000:1.0:1775764895.773982:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.773984:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.773992:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774001:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94040064 94044159) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774009:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774017:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774025:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.774033:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94040064, 94044160) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.774038:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22959 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.774056:0:18302:0:(rw26.c:858:ll_write_end()) pos 94040064, len 4096, copied 4096 00000080:00200000:1.0:1775764895.774061:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f57145d6[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.774064:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b0c42ef7 17ffffc0001001 3:0 ffff90c20b3e6380 22959 lru 00000080:00200000:1.0:1775764895.774070:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006c0cc08f 22959: 1< 0 - - > 2< 94040064 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6598656 - | + - + - > 5< + - + - | 0 - | 2479 - -> 00000080:00200000:1.0:1775764895.774072:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f57145d6 00000080:00200000:1.0:1775764895.774073:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.774075:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.774083:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.774091:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.774097:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.774104:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.774115:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.774127:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.774129:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.774139:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.774142:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.774144:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.774147:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774155:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.774162:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.774165:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.774169:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.774172:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22959, 22959] 00000080:00200000:1.0:1775764895.774182:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.774185:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.774213:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94044160 i_blocks 90112 00000080:00200000:1.0:1775764895.774217:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.774232:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94044160, count: 4096 00000080:00200000:1.0:1775764895.774243:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94044160, bytes: 4096 00000080:00200000:1.0:1775764895.774246:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.774250:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22961, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.774251:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94044160, 94048256) 0 0 00000080:00200000:1.0:1775764895.774253:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.774257:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22960, 22960] 00020000:00200000:1.0:1775764895.774259:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94044160 chunk: [94044160, 94048256) 94048256, 4096 00020000:00200000:1.0:1775764895.774261:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.774269:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22960, 22960] 00000008:00010000:1.0:1775764895.774272:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.774274:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.774280:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774286:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94044160 94048255) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774291:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774300:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774304:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.774308:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94044160, 94048256) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.774311:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22960 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.774324:0:18302:0:(rw26.c:858:ll_write_end()) pos 94044160, len 4096, copied 4096 00000080:00200000:1.0:1775764895.774327:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d6b9c1d1[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.774329:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000074dffc66 17ffffc0001001 3:0 ffff90c20b3e6620 22960 lru 00000080:00200000:1.0:1775764895.774334:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000790be1a0 22960: 1< 0 - - > 2< 94044160 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6594560 - | + - + - > 5< + - + - | 0 - | 2480 - -> 00000080:00200000:1.0:1775764895.774336:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d6b9c1d1 00000080:00200000:1.0:1775764895.774337:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.774339:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.774347:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.774354:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.774361:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.774368:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.774384:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.774388:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.774390:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.774400:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.774402:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.774405:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.774409:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774416:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.774421:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.774423:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.774425:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.774428:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22960, 22960] 00000080:00200000:1.0:1775764895.774435:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.774437:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.774457:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94048256 i_blocks 90112 00000080:00200000:1.0:1775764895.774459:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.774470:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94048256, count: 4096 00000080:00200000:1.0:1775764895.774476:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94048256, bytes: 4096 00000080:00200000:1.0:1775764895.774478:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.774480:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22962, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.774481:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94048256, 94052352) 0 0 00000080:00200000:1.0:1775764895.774483:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.774486:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22961, 22961] 00020000:00200000:1.0:1775764895.774489:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94048256 chunk: [94048256, 94052352) 94052352, 4096 00020000:00200000:1.0:1775764895.774490:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.774497:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22961, 22961] 00000008:00010000:1.0:1775764895.774500:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.774502:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.774507:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774513:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94048256 94052351) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774518:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774526:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774533:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.774541:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94048256, 94052352) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.774546:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22961 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.774561:0:18302:0:(rw26.c:858:ll_write_end()) pos 94048256, len 4096, copied 4096 00000080:00200000:1.0:1775764895.774566:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fdc7ff6d[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.774568:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000062f5d8ee 17ffffc0001001 3:0 ffff90c20b3e6e00 22961 lru 00000080:00200000:1.0:1775764895.774574:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d5763573 22961: 1< 0 - - > 2< 94048256 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6590464 - | + - + - > 5< + - + - | 0 - | 2481 - -> 00000080:00200000:1.0:1775764895.774576:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fdc7ff6d 00000080:00200000:1.0:1775764895.774577:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.774579:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.774587:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.774595:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.774602:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.774610:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.774621:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.774624:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.774626:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.774636:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.774638:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.774641:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.774644:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.774652:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.774659:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.774661:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.774665:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.774667:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22961, 22961] 00000080:00200000:1.0:1775764895.774679:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.774682:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.774704:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94052352 i_blocks 90112 00000080:00200000:1.0:1775764895.774706:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.774718:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94052352, count: 4096 00000080:00200000:1.0:1775764895.774724:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94052352, bytes: 4096 00000080:00200000:1.0:1775764895.774726:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.775108:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22963, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.775110:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94052352, 94056448) 0 0 00000080:00200000:1.0:1775764895.775113:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.775129:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22962, 22962] 00020000:00200000:1.0:1775764895.775133:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94052352 chunk: [94052352, 94056448) 94056448, 4096 00020000:00200000:1.0:1775764895.775135:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.775153:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22962, 22962] 00000008:00010000:1.0:1775764895.775159:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.775162:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.775171:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775181:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94052352 94056447) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775188:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775195:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775201:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.775210:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94052352, 94056448) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.775216:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22962 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.775240:0:18302:0:(rw26.c:858:ll_write_end()) pos 94052352, len 4096, copied 4096 00000080:00200000:1.0:1775764895.775245:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002867a8e4[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.775248:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000bcd83d9e 17ffffc0001001 3:0 ffff90c20b3e6540 22962 lru 00000080:00200000:1.0:1775764895.775253:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006bce3790 22962: 1< 0 - - > 2< 94052352 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6586368 - | + - + - > 5< + - + - | 0 - | 2482 - -> 00000080:00200000:1.0:1775764895.775255:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002867a8e4 00000080:00200000:1.0:1775764895.775256:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.775259:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.775268:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.775276:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.775283:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.775289:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.775302:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.775304:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.775307:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.775318:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.775321:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.775323:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.775328:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775335:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.775344:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.775346:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.775351:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.775353:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22962, 22962] 00000080:00200000:1.0:1775764895.775366:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.775369:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.775400:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94056448 i_blocks 90112 00000080:00200000:1.0:1775764895.775404:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.775421:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94056448, count: 4096 00000080:00200000:1.0:1775764895.775431:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94056448, bytes: 4096 00000080:00200000:1.0:1775764895.775434:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.775438:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22964, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.775440:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94056448, 94060544) 0 0 00000080:00200000:1.0:1775764895.775442:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.775447:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22963, 22963] 00020000:00200000:1.0:1775764895.775450:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94056448 chunk: [94056448, 94060544) 94060544, 4096 00020000:00200000:1.0:1775764895.775452:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.775462:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22963, 22963] 00000008:00010000:1.0:1775764895.775467:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.775469:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.775477:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775486:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94056448 94060543) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775495:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775503:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775510:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.775517:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94056448, 94060544) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.775643:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22963 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.775660:0:18302:0:(rw26.c:858:ll_write_end()) pos 94056448, len 4096, copied 4096 00000080:00200000:1.0:1775764895.775665:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000013873817[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.775668:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000184bcfa8 17ffffc0001001 3:0 ffff90c20b3e6460 22963 lru 00000080:00200000:1.0:1775764895.775673:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000aad50443 22963: 1< 0 - - > 2< 94056448 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6582272 - | + - + - > 5< + - + - | 0 - | 2483 - -> 00000080:00200000:1.0:1775764895.775675:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000013873817 00000080:00200000:1.0:1775764895.775676:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.775679:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.775687:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.775695:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.775702:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.775715:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.775825:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.775829:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.775832:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.775844:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.775846:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.775848:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.775853:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.775860:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.775867:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.775975:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.775982:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.775986:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22963, 22963] 00000080:00200000:1.0:1775764895.775999:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.776002:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.776036:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94060544 i_blocks 90112 00000080:00200000:1.0:1775764895.776041:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.776060:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94060544, count: 4096 00000080:00200000:1.0:1775764895.776070:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94060544, bytes: 4096 00000080:00200000:1.0:1775764895.776073:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.776077:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22965, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.776078:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94060544, 94064640) 0 0 00000080:00200000:1.0:1775764895.776080:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.776086:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22964, 22964] 00020000:00200000:1.0:1775764895.776090:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94060544 chunk: [94060544, 94064640) 94064640, 4096 00020000:00200000:1.0:1775764895.776092:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.776104:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22964, 22964] 00000008:00010000:1.0:1775764895.776110:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.776112:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.776129:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.776138:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94060544 94064639) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.776147:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.776154:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.776976:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.776991:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94060544, 94064640) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.777003:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22964 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.777038:0:18302:0:(rw26.c:858:ll_write_end()) pos 94060544, len 4096, copied 4096 00000080:00200000:1.0:1775764895.777044:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004554b224[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.777047:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a5429f65 17ffffc0001001 3:0 ffff90c20b3e6a80 22964 lru 00000080:00200000:1.0:1775764895.777053:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004d50a734 22964: 1< 0 - - > 2< 94060544 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6578176 - | + - + - > 5< + - + - | 0 - | 2484 - -> 00000080:00200000:1.0:1775764895.777055:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004554b224 00000080:00200000:1.0:1775764895.777057:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.777059:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.777072:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.777081:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.777088:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.777096:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.777110:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.777114:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.777126:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.777143:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.777146:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.777149:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.777154:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777162:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.777174:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.777177:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.777183:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.777187:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22964, 22964] 00000080:00200000:1.0:1775764895.777208:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.777211:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.777256:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94064640 i_blocks 90112 00000080:00200000:1.0:1775764895.777261:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.777283:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94064640, count: 4096 00000080:00200000:1.0:1775764895.777294:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94064640, bytes: 4096 00000080:00200000:1.0:1775764895.777297:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.777300:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22966, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.777301:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94064640, 94068736) 0 0 00000080:00200000:1.0:1775764895.777304:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.777311:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22965, 22965] 00020000:00200000:1.0:1775764895.777315:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94064640 chunk: [94064640, 94068736) 94068736, 4096 00020000:00200000:1.0:1775764895.777318:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.777327:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22965, 22965] 00000008:00010000:1.0:1775764895.777335:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.777339:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.777352:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777362:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94064640 94068735) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777371:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777379:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777392:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.777401:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94064640, 94068736) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.777406:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22965 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.777424:0:18302:0:(rw26.c:858:ll_write_end()) pos 94064640, len 4096, copied 4096 00000080:00200000:1.0:1775764895.777427:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000df8ba271[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.777429:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000264b92f1 17ffffc0001001 3:0 ffff90c20b3e6700 22965 lru 00000080:00200000:1.0:1775764895.777433:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cc2df07c 22965: 1< 0 - - > 2< 94064640 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6574080 - | + - + - > 5< + - + - | 0 - | 2485 - -> 00000080:00200000:1.0:1775764895.777434:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000df8ba271 00000080:00200000:1.0:1775764895.777435:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.777436:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.777442:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.777447:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.777451:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.777456:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.777463:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.777464:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.777466:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.777474:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.777476:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.777478:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.777480:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777485:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.777491:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.777492:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.777494:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.777496:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22965, 22965] 00000080:00200000:1.0:1775764895.777503:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.777505:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764895.777540:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 94068736 i_blocks 90112 00000080:00200000:1.0:1775764895.777544:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764895.777559:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 94068736, count: 4096 00000080:00200000:1.0:1775764895.777570:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 94068736, bytes: 4096 00000080:00200000:1.0:1775764895.777573:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764895.777576:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 22967, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764895.777578:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [94068736, 94072832) 0 0 00000080:00200000:1.0:1775764895.777580:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764895.777586:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [22966, 22966] 00020000:00200000:1.0:1775764895.777589:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 94068736 chunk: [94068736, 94072832) 94072832, 4096 00020000:00200000:1.0:1775764895.777592:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764895.777603:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [22966, 22966] 00000008:00010000:1.0:1775764895.777609:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764895.777611:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764895.777620:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777628:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (94068736 94072831) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 14/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777637:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777644:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777651:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.777657:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [94068736, 94072832) append 0 append lockpos 0 00000080:00200000:1.0:1775764895.777661:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 22966 of 0 to 4096 bytes 00000080:00200000:1.0:1775764895.777674:0:18302:0:(rw26.c:858:ll_write_end()) pos 94068736, len 4096, copied 4096 00000080:00200000:1.0:1775764895.777677:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000611110b8[3 00000000b99c1f5d 2 1 00000000a86a5a83] 00000080:00200000:1.0:1775764895.777679:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001e2605bd 17ffffc0001001 3:0 ffff90c20b3e6c40 22966 lru 00000080:00200000:1.0:1775764895.777682:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008f0e18ac 22966: 1< 0 - - > 2< 94068736 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 6569984 - | + - + - > 5< + - + - | 0 - | 2486 - -> 00000080:00200000:1.0:1775764895.777684:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000611110b8 00000080:00200000:1.0:1775764895.777685:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764895.777686:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764895.777692:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.777697:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009861ebc8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.777702:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764895.777706:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 00000000f6753c6d ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764895.777713:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764895.777714:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764895.777716:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764895.777722:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:1.0:1775764895.777724:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764895.777725:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764895.777853:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764895.777860:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764895.777866:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea998 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764895.777868:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764895.777902:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764895.777905:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [22966, 22966] 00000080:00200000:1.0:1775764895.777918:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764895.777921:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.013758:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f990b600 23599: 1< 0 - - > 2< 96661504 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3952640 - | + - + - > 5< + - + - | 0 - | 3119 - -> 00000080:00200000:3.0:1775764896.013761:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007d69a605 00000080:00200000:3.0:1775764896.013762:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.013764:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.013772:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.013779:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.013785:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.013792:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.013802:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.013805:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.013808:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.013818:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.013821:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.013822:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.013826:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.013833:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.013840:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.013843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.013848:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.013851:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23599, 23599] 00000080:00200000:3.0:1775764896.013861:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.013864:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.014010:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96665600 i_blocks 90112 00000080:00200000:3.0:1775764896.014015:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.014032:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96665600, count: 4096 00000080:00200000:3.0:1775764896.014042:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96665600, bytes: 4096 00000080:00200000:3.0:1775764896.014045:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.014049:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23601, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.014051:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96665600, 96669696) 0 0 00000080:00200000:3.0:1775764896.014053:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.014059:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23600, 23600] 00020000:00200000:3.0:1775764896.014062:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96665600 chunk: [96665600, 96669696) 96669696, 4096 00020000:00200000:3.0:1775764896.014064:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.014075:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23600, 23600] 00000008:00010000:3.0:1775764896.014080:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.014083:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.014091:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014099:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96665600 96669695) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014108:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014124:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014137:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.014146:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96665600, 96669696) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.014151:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23600 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.014169:0:18302:0:(rw26.c:858:ll_write_end()) pos 96665600, len 4096, copied 4096 00000080:00200000:3.0:1775764896.014174:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001980d925[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.014176:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004ad07607 17ffffc0001001 3:0 ffff90c2128092a0 23600 lru 00000080:00200000:3.0:1775764896.014182:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f6f888fe 23600: 1< 0 - - > 2< 96665600 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3948544 - | + - + - > 5< + - + - | 0 - | 3120 - -> 00000080:00200000:3.0:1775764896.014184:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001980d925 00000080:00200000:3.0:1775764896.014185:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.014187:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.014196:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.014204:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.014210:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.014217:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.014228:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.014232:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.014234:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.014246:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.014249:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.014251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.014255:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014263:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.014272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.014274:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.014278:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.014281:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23600, 23600] 00000080:00200000:3.0:1775764896.014293:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.014296:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.014326:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96669696 i_blocks 90112 00000080:00200000:3.0:1775764896.014329:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.014346:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96669696, count: 4096 00000080:00200000:3.0:1775764896.014356:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96669696, bytes: 4096 00000080:00200000:3.0:1775764896.014359:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.014362:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23602, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.014364:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96669696, 96673792) 0 0 00000080:00200000:3.0:1775764896.014366:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.014372:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23601, 23601] 00020000:00200000:3.0:1775764896.014375:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96669696 chunk: [96669696, 96673792) 96673792, 4096 00020000:00200000:3.0:1775764896.014377:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.014389:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23601, 23601] 00000008:00010000:3.0:1775764896.014394:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.014397:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.014405:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014414:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96669696 96673791) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014422:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014429:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014436:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.014528:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96669696, 96673792) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.014535:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23601 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.014551:0:18302:0:(rw26.c:858:ll_write_end()) pos 96669696, len 4096, copied 4096 00000080:00200000:3.0:1775764896.014556:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000af0ded36[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.014559:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000dcf2db1c 17ffffc0001001 3:0 ffff90c212809d20 23601 lru 00000080:00200000:3.0:1775764896.014564:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a3079c58 23601: 1< 0 - - > 2< 96669696 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3944448 - | + - + - > 5< + - + - | 0 - | 3121 - -> 00000080:00200000:3.0:1775764896.014566:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000af0ded36 00000080:00200000:3.0:1775764896.014567:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.014569:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.014578:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.014586:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.014593:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.014601:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.014613:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.014616:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.014618:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.014628:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.014631:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.014633:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.014638:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014646:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.014653:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.014656:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.014660:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.014663:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23601, 23601] 00000080:00200000:3.0:1775764896.014674:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.014677:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.014708:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96673792 i_blocks 90112 00000080:00200000:3.0:1775764896.014712:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.014729:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96673792, count: 4096 00000080:00200000:3.0:1775764896.014740:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96673792, bytes: 4096 00000080:00200000:3.0:1775764896.014743:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.014747:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23603, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.014748:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96673792, 96677888) 0 0 00000080:00200000:3.0:1775764896.014751:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.014756:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23602, 23602] 00020000:00200000:3.0:1775764896.014759:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96673792 chunk: [96673792, 96677888) 96677888, 4096 00020000:00200000:3.0:1775764896.014762:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.014774:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23602, 23602] 00000008:00010000:3.0:1775764896.014779:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.014782:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.014789:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014798:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96673792 96677887) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014807:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014814:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.014821:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.014829:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96673792, 96677888) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.014834:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23602 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.014851:0:18302:0:(rw26.c:858:ll_write_end()) pos 96673792, len 4096, copied 4096 00000080:00200000:3.0:1775764896.014856:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000043dd7fc1[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.014859:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000495d0347 17ffffc0001001 3:0 ffff90c212809380 23602 lru 00000080:00200000:3.0:1775764896.014864:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e5e374d9 23602: 1< 0 - - > 2< 96673792 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3940352 - | + - + - > 5< + - + - | 0 - | 3122 - -> 00000080:00200000:3.0:1775764896.014866:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000043dd7fc1 00000080:00200000:3.0:1775764896.014867:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.014968:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.014979:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.014987:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.014994:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.015002:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.015013:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.015016:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.015019:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.015029:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.015032:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.015034:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.015038:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015046:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.015056:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.015058:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.015063:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.015066:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23602, 23602] 00000080:00200000:3.0:1775764896.015076:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.015080:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.015110:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96677888 i_blocks 90112 00000080:00200000:3.0:1775764896.015114:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.015141:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96677888, count: 4096 00000080:00200000:3.0:1775764896.015152:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96677888, bytes: 4096 00000080:00200000:3.0:1775764896.015154:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.015158:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23604, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.015160:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96677888, 96681984) 0 0 00000080:00200000:3.0:1775764896.015162:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.015168:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23603, 23603] 00020000:00200000:3.0:1775764896.015171:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96677888 chunk: [96677888, 96681984) 96681984, 4096 00020000:00200000:3.0:1775764896.015173:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.015185:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23603, 23603] 00000008:00010000:3.0:1775764896.015190:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.015193:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.015201:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015210:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96677888 96681983) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015219:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015232:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015238:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.015245:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96677888, 96681984) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.015250:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23603 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.015269:0:18302:0:(rw26.c:858:ll_write_end()) pos 96677888, len 4096, copied 4096 00000080:00200000:3.0:1775764896.015273:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001f3e0574[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.015276:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000010b7529f 17ffffc0001001 3:0 ffff90c212809620 23603 lru 00000080:00200000:3.0:1775764896.015280:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000037cfe4cb 23603: 1< 0 - - > 2< 96677888 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3936256 - | + - + - > 5< + - + - | 0 - | 3123 - -> 00000080:00200000:3.0:1775764896.015281:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001f3e0574 00000080:00200000:3.0:1775764896.015282:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.015285:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.015292:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.015300:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.015307:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.015315:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.015332:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.015335:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.015338:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.015347:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.015350:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.015352:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.015356:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015364:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.015371:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.015373:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.015376:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.015379:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23603, 23603] 00000080:00200000:3.0:1775764896.015389:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.015392:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.015422:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96681984 i_blocks 90112 00000080:00200000:3.0:1775764896.015425:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.015532:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96681984, count: 4096 00000080:00200000:3.0:1775764896.015547:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96681984, bytes: 4096 00000080:00200000:3.0:1775764896.015550:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.015554:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23605, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.015555:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96681984, 96686080) 0 0 00000080:00200000:3.0:1775764896.015558:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.015563:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23604, 23604] 00020000:00200000:3.0:1775764896.015567:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96681984 chunk: [96681984, 96686080) 96686080, 4096 00020000:00200000:3.0:1775764896.015569:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.015582:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23604, 23604] 00000008:00010000:3.0:1775764896.015588:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.015591:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.015599:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015608:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96681984 96686079) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015616:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015622:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015628:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.015635:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96681984, 96686080) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.015640:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23604 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.015657:0:18302:0:(rw26.c:858:ll_write_end()) pos 96681984, len 4096, copied 4096 00000080:00200000:3.0:1775764896.015661:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d86e5a8e[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.015664:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000036b9cd0d 17ffffc0001001 3:0 ffff90c212809e00 23604 lru 00000080:00200000:3.0:1775764896.015669:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002f6e22f4 23604: 1< 0 - - > 2< 96681984 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3932160 - | + - + - > 5< + - + - | 0 - | 3124 - -> 00000080:00200000:3.0:1775764896.015670:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d86e5a8e 00000080:00200000:3.0:1775764896.015671:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.015674:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.015683:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.015691:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.015697:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.015704:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.015716:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.015718:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.015721:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.015731:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.015734:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.015736:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.015740:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.015748:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.015755:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.015757:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.015762:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.015764:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23604, 23604] 00000080:00200000:3.0:1775764896.015774:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.015777:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.015807:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96686080 i_blocks 90112 00000080:00200000:3.0:1775764896.015810:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.015826:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96686080, count: 4096 00000080:00200000:3.0:1775764896.015835:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96686080, bytes: 4096 00000080:00200000:3.0:1775764896.015838:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.015842:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23606, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.015843:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96686080, 96690176) 0 0 00000080:00200000:3.0:1775764896.015845:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.015851:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23605, 23605] 00020000:00200000:3.0:1775764896.015854:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96686080 chunk: [96686080, 96690176) 96690176, 4096 00020000:00200000:3.0:1775764896.015856:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.015979:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23605, 23605] 00000008:00010000:3.0:1775764896.015988:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.015991:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.015999:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016008:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96686080 96690175) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016016:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016024:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016031:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.016039:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96686080, 96690176) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.016044:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23605 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.016060:0:18302:0:(rw26.c:858:ll_write_end()) pos 96686080, len 4096, copied 4096 00000080:00200000:3.0:1775764896.016064:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000835e950f[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.016067:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000061a55f6d 17ffffc0001001 3:0 ffff90c212809540 23605 lru 00000080:00200000:3.0:1775764896.016072:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009a7309d8 23605: 1< 0 - - > 2< 96686080 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3928064 - | + - + - > 5< + - + - | 0 - | 3125 - -> 00000080:00200000:3.0:1775764896.016075:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000835e950f 00000080:00200000:3.0:1775764896.016076:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.016078:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.016086:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.016094:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.016101:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.016108:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.016131:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.016135:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.016137:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.016147:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.016150:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.016152:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.016157:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016164:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.016172:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.016174:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.016178:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.016181:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23605, 23605] 00000080:00200000:3.0:1775764896.016192:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.016196:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.016226:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96690176 i_blocks 90112 00000080:00200000:3.0:1775764896.016230:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.016246:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96690176, count: 4096 00000080:00200000:3.0:1775764896.016257:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96690176, bytes: 4096 00000080:00200000:3.0:1775764896.016260:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.016264:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23607, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.016266:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96690176, 96694272) 0 0 00000080:00200000:3.0:1775764896.016268:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.016274:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23606, 23606] 00020000:00200000:3.0:1775764896.016277:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96690176 chunk: [96690176, 96694272) 96694272, 4096 00020000:00200000:3.0:1775764896.016279:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.016292:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23606, 23606] 00000008:00010000:3.0:1775764896.016298:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.016300:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.016307:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016316:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96690176 96694271) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016325:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016333:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016340:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.016348:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96690176, 96694272) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.016355:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23606 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.016374:0:18302:0:(rw26.c:858:ll_write_end()) pos 96690176, len 4096, copied 4096 00000080:00200000:3.0:1775764896.016378:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a05d3684[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.016381:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000091c98936 17ffffc0001001 3:0 ffff90c212809460 23606 lru 00000080:00200000:3.0:1775764896.016387:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000042370af5 23606: 1< 0 - - > 2< 96690176 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3923968 - | + - + - > 5< + - + - | 0 - | 3126 - -> 00000080:00200000:3.0:1775764896.016389:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a05d3684 00000080:00200000:3.0:1775764896.016390:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.016392:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.016400:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.016408:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.016415:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.016428:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.016525:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.016530:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.016532:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.016544:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.016547:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.016550:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.016554:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016562:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.016571:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.016573:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.016578:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.016581:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23606, 23606] 00000080:00200000:3.0:1775764896.016592:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.016595:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.016627:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96694272 i_blocks 90112 00000080:00200000:3.0:1775764896.016631:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.016648:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96694272, count: 4096 00000080:00200000:3.0:1775764896.016659:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96694272, bytes: 4096 00000080:00200000:3.0:1775764896.016663:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.016667:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23608, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.016668:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96694272, 96698368) 0 0 00000080:00200000:3.0:1775764896.016671:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.016677:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23607, 23607] 00020000:00200000:3.0:1775764896.016681:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96694272 chunk: [96694272, 96698368) 96698368, 4096 00020000:00200000:3.0:1775764896.016683:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.016696:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23607, 23607] 00000008:00010000:3.0:1775764896.016701:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.016703:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.016711:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016720:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96694272 96698367) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016729:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016737:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.016745:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.016753:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96694272, 96698368) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.016759:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23607 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.016776:0:18302:0:(rw26.c:858:ll_write_end()) pos 96694272, len 4096, copied 4096 00000080:00200000:3.0:1775764896.016780:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000061f7f2a0[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.016783:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002a695e01 17ffffc0001001 3:0 ffff90c212809a80 23607 lru 00000080:00200000:3.0:1775764896.016789:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000076ea6556 23607: 1< 0 - - > 2< 96694272 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3919872 - | + - + - > 5< + - + - | 0 - | 3127 - -> 00000080:00200000:3.0:1775764896.016791:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000061f7f2a0 00000080:00200000:3.0:1775764896.016793:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.016795:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.016802:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.016810:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.016817:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.016824:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.016836:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.016840:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.016842:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.016852:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.016855:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.016857:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.016861:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017051:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.017061:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.017064:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.017069:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.017072:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23607, 23607] 00000080:00200000:3.0:1775764896.017082:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.017085:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.017126:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96698368 i_blocks 90112 00000080:00200000:3.0:1775764896.017130:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.017149:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96698368, count: 4096 00000080:00200000:3.0:1775764896.017160:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96698368, bytes: 4096 00000080:00200000:3.0:1775764896.017162:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.017166:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23609, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.017168:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96698368, 96702464) 0 0 00000080:00200000:3.0:1775764896.017170:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.017176:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23608, 23608] 00020000:00200000:3.0:1775764896.017179:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96698368 chunk: [96698368, 96702464) 96702464, 4096 00020000:00200000:3.0:1775764896.017181:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.017192:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23608, 23608] 00000008:00010000:3.0:1775764896.017198:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.017200:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.017208:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017217:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96698368 96702463) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017225:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017233:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017245:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.017253:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96698368, 96702464) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.017258:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23608 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.017274:0:18302:0:(rw26.c:858:ll_write_end()) pos 96698368, len 4096, copied 4096 00000080:00200000:3.0:1775764896.017278:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cb67f364[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.017281:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000090d6bf54 17ffffc0001001 3:0 ffff90c212809700 23608 lru 00000080:00200000:3.0:1775764896.017287:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000014dbbeb 23608: 1< 0 - - > 2< 96698368 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3915776 - | + - + - > 5< + - + - | 0 - | 3128 - -> 00000080:00200000:3.0:1775764896.017289:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cb67f364 00000080:00200000:3.0:1775764896.017290:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.017292:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.017300:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.017308:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.017315:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.017323:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.017335:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.017338:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.017341:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.017351:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.017354:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.017357:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.017361:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017370:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.017378:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.017381:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.017385:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.017388:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23608, 23608] 00000080:00200000:3.0:1775764896.017399:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.017402:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.017433:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96702464 i_blocks 90112 00000080:00200000:3.0:1775764896.017436:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.017538:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96702464, count: 4096 00000080:00200000:3.0:1775764896.017551:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96702464, bytes: 4096 00000080:00200000:3.0:1775764896.017553:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.017557:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23610, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.017559:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96702464, 96706560) 0 0 00000080:00200000:3.0:1775764896.017561:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.017567:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23609, 23609] 00020000:00200000:3.0:1775764896.017570:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96702464 chunk: [96702464, 96706560) 96706560, 4096 00020000:00200000:3.0:1775764896.017573:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.017585:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23609, 23609] 00000008:00010000:3.0:1775764896.017591:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.017593:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.017603:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017612:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96702464 96706559) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017621:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017628:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017635:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.017643:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96702464, 96706560) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.017648:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23609 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.017667:0:18302:0:(rw26.c:858:ll_write_end()) pos 96702464, len 4096, copied 4096 00000080:00200000:3.0:1775764896.017671:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000bdc5117a[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.017674:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000632361a2 17ffffc0001001 3:0 ffff90c212809c40 23609 lru 00000080:00200000:3.0:1775764896.017679:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ff845f19 23609: 1< 0 - - > 2< 96702464 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3911680 - | + - + - > 5< + - + - | 0 - | 3129 - -> 00000080:00200000:3.0:1775764896.017682:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000bdc5117a 00000080:00200000:3.0:1775764896.017683:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.017685:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.017694:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.017701:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.017709:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.017717:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.017729:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.017732:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.017734:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.017744:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.017747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.017749:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.017754:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.017762:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.017770:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.017773:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.017777:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.017779:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23609, 23609] 00000080:00200000:3.0:1775764896.017790:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.017793:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.017819:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96706560 i_blocks 90112 00000080:00200000:3.0:1775764896.017823:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.017838:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96706560, count: 4096 00000080:00200000:3.0:1775764896.017847:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96706560, bytes: 4096 00000080:00200000:3.0:1775764896.017850:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.017853:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23611, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.017855:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96706560, 96710656) 0 0 00000080:00200000:3.0:1775764896.017857:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.017862:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23610, 23610] 00020000:00200000:3.0:1775764896.017865:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96706560 chunk: [96706560, 96710656) 96710656, 4096 00020000:00200000:3.0:1775764896.017867:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.017983:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23610, 23610] 00000008:00010000:3.0:1775764896.017990:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.017992:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.018000:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018009:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96706560 96710655) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018018:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018026:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018033:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.018041:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96706560, 96710656) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.018046:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23610 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.018081:0:18302:0:(rw26.c:858:ll_write_end()) pos 96706560, len 4096, copied 4096 00000080:00200000:3.0:1775764896.018086:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000271f26bf[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.018089:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000600d8805 17ffffc0001021 2:0 ffff90c21280a380 23610 lru 00000080:00200000:3.0:1775764896.018094:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003582c0c1 23610: 1< 0 - - > 2< 96706560 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3907584 - | + - + - > 5< + - + - | 0 - | 3130 - -> 00000080:00200000:3.0:1775764896.018096:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000271f26bf 00000080:00200000:3.0:1775764896.018097:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.018099:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.018108:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.018127:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.018134:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.018141:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.018154:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.018157:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.018159:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.018169:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.018172:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.018174:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.018178:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018186:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.018194:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.018197:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.018200:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.018203:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23610, 23610] 00000080:00200000:3.0:1775764896.018214:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.018217:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.018247:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96710656 i_blocks 90112 00000080:00200000:3.0:1775764896.018251:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.018266:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96710656, count: 4096 00000080:00200000:3.0:1775764896.018276:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96710656, bytes: 4096 00000080:00200000:3.0:1775764896.018278:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.018281:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23612, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.018283:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96710656, 96714752) 0 0 00000080:00200000:3.0:1775764896.018285:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.018291:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23611, 23611] 00020000:00200000:3.0:1775764896.018294:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96710656 chunk: [96710656, 96714752) 96714752, 4096 00020000:00200000:3.0:1775764896.018296:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.018307:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23611, 23611] 00000008:00010000:3.0:1775764896.018312:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.018315:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.018322:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018331:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96710656 96714751) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018339:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018351:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018358:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.018365:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96710656, 96714752) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.018370:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23611 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.018385:0:18302:0:(rw26.c:858:ll_write_end()) pos 96710656, len 4096, copied 4096 00000080:00200000:3.0:1775764896.018389:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f84beed8[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.018392:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001df7a469 17ffffc0001001 3:0 ffff90c21280a620 23611 lru 00000080:00200000:3.0:1775764896.018397:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a9e2cbb0 23611: 1< 0 - - > 2< 96710656 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3903488 - | + - + - > 5< + - + - | 0 - | 3131 - -> 00000080:00200000:3.0:1775764896.018399:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f84beed8 00000080:00200000:3.0:1775764896.018400:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.018402:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.018409:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.018418:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.018424:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.018432:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.018533:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.018536:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.018539:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.018550:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.018553:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.018556:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.018560:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018568:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.018575:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.018577:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.018581:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.018584:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23611, 23611] 00000080:00200000:3.0:1775764896.018595:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.018598:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.018630:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96714752 i_blocks 90112 00000080:00200000:3.0:1775764896.018633:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.018649:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96714752, count: 4096 00000080:00200000:3.0:1775764896.018661:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96714752, bytes: 4096 00000080:00200000:3.0:1775764896.018663:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.018667:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23613, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.018669:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96714752, 96718848) 0 0 00000080:00200000:3.0:1775764896.018671:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.018676:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23612, 23612] 00020000:00200000:3.0:1775764896.018680:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96714752 chunk: [96714752, 96718848) 96718848, 4096 00020000:00200000:3.0:1775764896.018682:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.018694:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23612, 23612] 00000008:00010000:3.0:1775764896.018700:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.018702:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.018710:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018719:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96714752 96718847) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018728:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018736:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.018742:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.018750:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96714752, 96718848) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.018756:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23612 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.018773:0:18302:0:(rw26.c:858:ll_write_end()) pos 96714752, len 4096, copied 4096 00000080:00200000:3.0:1775764896.018778:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004e8b1c6d[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.018781:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000529f3bb 17ffffc0001001 3:0 ffff90c21280ae00 23612 lru 00000080:00200000:3.0:1775764896.018787:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000750e1b0c 23612: 1< 0 - - > 2< 96714752 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3899392 - | + - + - > 5< + - + - | 0 - | 3132 - -> 00000080:00200000:3.0:1775764896.018789:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004e8b1c6d 00000080:00200000:3.0:1775764896.018790:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.018793:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.018801:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.018810:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.018817:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.018825:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.018836:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.018839:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.018841:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.018851:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.018854:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.018857:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.018861:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019034:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.019046:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.019049:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.019054:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.019057:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23612, 23612] 00000080:00200000:3.0:1775764896.019069:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.019072:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.019104:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96718848 i_blocks 90112 00000080:00200000:3.0:1775764896.019108:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.019137:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96718848, count: 4096 00000080:00200000:3.0:1775764896.019148:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96718848, bytes: 4096 00000080:00200000:3.0:1775764896.019151:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.019155:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23614, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.019156:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96718848, 96722944) 0 0 00000080:00200000:3.0:1775764896.019158:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.019164:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23613, 23613] 00020000:00200000:3.0:1775764896.019167:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96718848 chunk: [96718848, 96722944) 96722944, 4096 00020000:00200000:3.0:1775764896.019169:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.019180:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23613, 23613] 00000008:00010000:3.0:1775764896.019185:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.019188:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.019195:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019203:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96718848 96722943) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019211:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019217:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019224:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.019233:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96718848, 96722944) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.019237:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23613 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.019253:0:18302:0:(rw26.c:858:ll_write_end()) pos 96718848, len 4096, copied 4096 00000080:00200000:3.0:1775764896.019258:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000553bfa8b[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.019260:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000968b20a3 17ffffc0001001 3:0 ffff90c21280a540 23613 lru 00000080:00200000:3.0:1775764896.019265:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b38f8c30 23613: 1< 0 - - > 2< 96718848 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3895296 - | + - + - > 5< + - + - | 0 - | 3133 - -> 00000080:00200000:3.0:1775764896.019267:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000553bfa8b 00000080:00200000:3.0:1775764896.019269:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.019271:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.019278:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.019285:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.019292:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.019300:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.019312:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.019314:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.019316:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.019327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.019329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.019332:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.019336:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019344:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.019352:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.019354:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.019357:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.019360:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23613, 23613] 00000080:00200000:3.0:1775764896.019370:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.019374:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.019402:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96722944 i_blocks 90112 00000080:00200000:3.0:1775764896.019406:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.019422:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96722944, count: 4096 00000080:00200000:3.0:1775764896.019432:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96722944, bytes: 4096 00000080:00200000:3.0:1775764896.019435:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.019524:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23615, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.019526:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96722944, 96727040) 0 0 00000080:00200000:3.0:1775764896.019529:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.019536:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23614, 23614] 00020000:00200000:3.0:1775764896.019539:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96722944 chunk: [96722944, 96727040) 96727040, 4096 00020000:00200000:3.0:1775764896.019541:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.019556:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23614, 23614] 00000008:00010000:3.0:1775764896.019562:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.019564:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.019573:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019582:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96722944 96727039) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019592:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019599:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019605:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.019613:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96722944, 96727040) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.019618:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23614 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.019634:0:18302:0:(rw26.c:858:ll_write_end()) pos 96722944, len 4096, copied 4096 00000080:00200000:3.0:1775764896.019638:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000893b3a58[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.019641:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006f0c0bbc 17ffffc0001001 3:0 ffff90c21280a460 23614 lru 00000080:00200000:3.0:1775764896.019646:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007013d32a 23614: 1< 0 - - > 2< 96722944 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3891200 - | + - + - > 5< + - + - | 0 - | 3134 - -> 00000080:00200000:3.0:1775764896.019649:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000893b3a58 00000080:00200000:3.0:1775764896.019650:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.019652:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.019659:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.019666:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.019673:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.019685:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.019697:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.019700:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.019702:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.019713:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.019715:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.019718:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.019722:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019730:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.019739:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.019741:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.019746:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.019749:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23614, 23614] 00000080:00200000:3.0:1775764896.019759:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.019762:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.019791:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96727040 i_blocks 90112 00000080:00200000:3.0:1775764896.019795:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.019811:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96727040, count: 4096 00000080:00200000:3.0:1775764896.019822:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96727040, bytes: 4096 00000080:00200000:3.0:1775764896.019824:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.019829:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23616, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.019830:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96727040, 96731136) 0 0 00000080:00200000:3.0:1775764896.019833:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.019838:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23615, 23615] 00020000:00200000:3.0:1775764896.019841:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96727040 chunk: [96727040, 96731136) 96731136, 4096 00020000:00200000:3.0:1775764896.019843:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.019854:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23615, 23615] 00000008:00010000:3.0:1775764896.019860:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.019862:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.019891:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019979:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96727040 96731135) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019989:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.019997:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020004:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.020013:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96727040, 96731136) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.020019:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23615 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.020037:0:18302:0:(rw26.c:858:ll_write_end()) pos 96727040, len 4096, copied 4096 00000080:00200000:3.0:1775764896.020041:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d0a74b56[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.020044:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000dd09915e 17ffffc0001001 3:0 ffff90c21280aa80 23615 lru 00000080:00200000:3.0:1775764896.020049:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000eb0cdd39 23615: 1< 0 - - > 2< 96727040 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3887104 - | + - + - > 5< + - + - | 0 - | 3135 - -> 00000080:00200000:3.0:1775764896.020051:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d0a74b56 00000080:00200000:3.0:1775764896.020053:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.020055:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.020062:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.020070:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.020077:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.020084:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.020094:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.020097:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.020099:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.020108:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.020110:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.020112:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.020129:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020137:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.020145:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.020148:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.020153:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.020156:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23615, 23615] 00000080:00200000:3.0:1775764896.020167:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.020169:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.020199:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96731136 i_blocks 90112 00000080:00200000:3.0:1775764896.020202:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.020219:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96731136, count: 4096 00000080:00200000:3.0:1775764896.020230:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96731136, bytes: 4096 00000080:00200000:3.0:1775764896.020232:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.020236:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23617, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.020237:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96731136, 96735232) 0 0 00000080:00200000:3.0:1775764896.020240:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.020245:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23616, 23616] 00020000:00200000:3.0:1775764896.020248:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96731136 chunk: [96731136, 96735232) 96735232, 4096 00020000:00200000:3.0:1775764896.020250:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.020261:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23616, 23616] 00000008:00010000:3.0:1775764896.020266:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.020269:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.020276:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020284:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96731136 96735231) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020292:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020300:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020311:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.020319:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96731136, 96735232) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.020323:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23616 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.020346:0:18302:0:(rw26.c:858:ll_write_end()) pos 96731136, len 4096, copied 4096 00000080:00200000:3.0:1775764896.020350:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000050907422[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.020352:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000659d7bed 17ffffc0001001 3:0 ffff90c21280a700 23616 lru 00000080:00200000:3.0:1775764896.020357:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b32b71d9 23616: 1< 0 - - > 2< 96731136 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3883008 - | + - + - > 5< + - + - | 0 - | 3136 - -> 00000080:00200000:3.0:1775764896.020359:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000050907422 00000080:00200000:3.0:1775764896.020360:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.020362:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.020369:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.020376:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.020382:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.020389:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.020648:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.020652:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.020655:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.020666:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.020668:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.020671:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.020675:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020683:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.020691:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.020694:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.020698:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.020701:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23616, 23616] 00000080:00200000:3.0:1775764896.020713:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.020715:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.020747:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96735232 i_blocks 90112 00000080:00200000:3.0:1775764896.020751:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.020767:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96735232, count: 4096 00000080:00200000:3.0:1775764896.020778:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96735232, bytes: 4096 00000080:00200000:3.0:1775764896.020780:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.020784:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23618, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.020786:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96735232, 96739328) 0 0 00000080:00200000:3.0:1775764896.020788:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.020794:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23617, 23617] 00020000:00200000:3.0:1775764896.020797:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96735232 chunk: [96735232, 96739328) 96739328, 4096 00020000:00200000:3.0:1775764896.020799:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.020810:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23617, 23617] 00000008:00010000:3.0:1775764896.020816:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.020819:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.020827:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020836:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96735232 96739327) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020844:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020852:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.020859:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.020866:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96735232, 96739328) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.020892:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23617 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.020990:0:18302:0:(rw26.c:858:ll_write_end()) pos 96735232, len 4096, copied 4096 00000080:00200000:3.0:1775764896.020994:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b807f9bd[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.020997:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000924b145b 17ffffc0001001 3:0 ffff90c21280ac40 23617 lru 00000080:00200000:3.0:1775764896.021003:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000009e464b3 23617: 1< 0 - - > 2< 96735232 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3878912 - | + - + - > 5< + - + - | 0 - | 3137 - -> 00000080:00200000:3.0:1775764896.021005:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b807f9bd 00000080:00200000:3.0:1775764896.021006:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.021008:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.021017:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.021025:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.021033:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.021041:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.021052:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.021054:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.021057:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.021067:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.021069:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.021071:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.021075:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021082:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.021089:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.021091:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.021095:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.021098:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23617, 23617] 00000080:00200000:3.0:1775764896.021108:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.021111:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.021150:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96739328 i_blocks 90112 00000080:00200000:3.0:1775764896.021154:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.021169:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96739328, count: 4096 00000080:00200000:3.0:1775764896.021178:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96739328, bytes: 4096 00000080:00200000:3.0:1775764896.021181:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.021184:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23619, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.021186:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96739328, 96743424) 0 0 00000080:00200000:3.0:1775764896.021188:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.021194:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23618, 23618] 00020000:00200000:3.0:1775764896.021197:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96739328 chunk: [96739328, 96743424) 96743424, 4096 00020000:00200000:3.0:1775764896.021199:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.021209:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23618, 23618] 00000008:00010000:3.0:1775764896.021214:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.021217:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.021225:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021234:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96739328 96743423) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021243:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021250:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021257:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.021265:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96739328, 96743424) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.021269:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23618 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.021287:0:18302:0:(rw26.c:858:ll_write_end()) pos 96739328, len 4096, copied 4096 00000080:00200000:3.0:1775764896.021291:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005dd3bd6b[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.021294:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003eb75b4a 17ffffc0001001 3:0 ffff90c21280a8c0 23618 lru 00000080:00200000:3.0:1775764896.021299:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a5ac3630 23618: 1< 0 - - > 2< 96739328 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3874816 - | + - + - > 5< + - + - | 0 - | 3138 - -> 00000080:00200000:3.0:1775764896.021302:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005dd3bd6b 00000080:00200000:3.0:1775764896.021303:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.021305:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.021312:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.021319:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.021326:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.021333:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.021345:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.021347:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.021349:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.021358:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.021360:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.021362:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.021366:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021374:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.021382:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.021385:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.021389:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.021392:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23618, 23618] 00000080:00200000:3.0:1775764896.021402:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.021405:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.021433:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96743424 i_blocks 90112 00000080:00200000:3.0:1775764896.021437:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.021530:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96743424, count: 4096 00000080:00200000:3.0:1775764896.021542:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96743424, bytes: 4096 00000080:00200000:3.0:1775764896.021544:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.021548:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23620, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.021549:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96743424, 96747520) 0 0 00000080:00200000:3.0:1775764896.021551:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.021556:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23619, 23619] 00020000:00200000:3.0:1775764896.021559:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96743424 chunk: [96743424, 96747520) 96747520, 4096 00020000:00200000:3.0:1775764896.021562:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.021572:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23619, 23619] 00000008:00010000:3.0:1775764896.021577:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.021579:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.021587:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021595:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96743424 96747519) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021603:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021615:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021621:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.021628:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96743424, 96747520) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.021633:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23619 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.021649:0:18302:0:(rw26.c:858:ll_write_end()) pos 96743424, len 4096, copied 4096 00000080:00200000:3.0:1775764896.021653:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000081daf31[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.021656:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005dde656c 17ffffc0001001 3:0 ffff90c21280ab60 23619 lru 00000080:00200000:3.0:1775764896.021661:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004c5c89f6 23619: 1< 0 - - > 2< 96743424 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3870720 - | + - + - > 5< + - + - | 0 - | 3139 - -> 00000080:00200000:3.0:1775764896.021663:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000081daf31 00000080:00200000:3.0:1775764896.021664:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.021666:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.021673:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.021681:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.021687:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.021694:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.021709:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.021712:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.021714:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.021723:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.021725:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.021728:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.021731:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021739:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.021747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.021749:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.021753:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.021756:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23619, 23619] 00000080:00200000:3.0:1775764896.021766:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.021769:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.021796:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96747520 i_blocks 90112 00000080:00200000:3.0:1775764896.021800:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.021814:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96747520, count: 4096 00000080:00200000:3.0:1775764896.021824:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96747520, bytes: 4096 00000080:00200000:3.0:1775764896.021826:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.021830:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23621, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.021831:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96747520, 96751616) 0 0 00000080:00200000:3.0:1775764896.021834:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.021838:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23620, 23620] 00020000:00200000:3.0:1775764896.021842:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96747520 chunk: [96747520, 96751616) 96751616, 4096 00020000:00200000:3.0:1775764896.021844:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.021855:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23620, 23620] 00000008:00010000:3.0:1775764896.021859:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.021862:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.021891:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021976:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96747520 96751615) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021985:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.021992:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022000:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.022008:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96747520, 96751616) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.022015:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23620 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.022031:0:18302:0:(rw26.c:858:ll_write_end()) pos 96747520, len 4096, copied 4096 00000080:00200000:3.0:1775764896.022035:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000db2f6935[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.022037:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000030b9be5 17ffffc0001001 3:0 ffff90c21280a000 23620 lru 00000080:00200000:3.0:1775764896.022042:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a24af959 23620: 1< 0 - - > 2< 96747520 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3866624 - | + - + - > 5< + - + - | 0 - | 3140 - -> 00000080:00200000:3.0:1775764896.022045:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000db2f6935 00000080:00200000:3.0:1775764896.022046:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.022048:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.022056:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.022063:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.022070:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.022077:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.022089:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.022092:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.022094:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.022104:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.022107:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.022109:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.022113:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022131:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.022139:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.022141:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.022146:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.022149:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23620, 23620] 00000080:00200000:3.0:1775764896.022161:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.022163:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.022193:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96751616 i_blocks 90112 00000080:00200000:3.0:1775764896.022197:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.022212:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96751616, count: 4096 00000080:00200000:3.0:1775764896.022223:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96751616, bytes: 4096 00000080:00200000:3.0:1775764896.022226:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.022229:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23622, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.022231:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96751616, 96755712) 0 0 00000080:00200000:3.0:1775764896.022233:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.022238:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23621, 23621] 00020000:00200000:3.0:1775764896.022241:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96751616 chunk: [96751616, 96755712) 96755712, 4096 00020000:00200000:3.0:1775764896.022244:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.022254:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23621, 23621] 00000008:00010000:3.0:1775764896.022259:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.022262:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.022269:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022278:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96751616 96755711) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022286:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022294:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022301:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.022308:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96751616, 96755712) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.022313:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23621 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.022330:0:18302:0:(rw26.c:858:ll_write_end()) pos 96751616, len 4096, copied 4096 00000080:00200000:3.0:1775764896.022335:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000395e5be2[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.022337:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004225c69d 17ffffc0001001 3:0 ffff90c21280aee0 23621 lru 00000080:00200000:3.0:1775764896.022343:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000028303cc9 23621: 1< 0 - - > 2< 96751616 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3862528 - | + - + - > 5< + - + - | 0 - | 3141 - -> 00000080:00200000:3.0:1775764896.022345:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000395e5be2 00000080:00200000:3.0:1775764896.022346:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.022348:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.022355:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.022363:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.022369:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.022376:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.022387:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.022390:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.022392:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.022402:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.022405:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.022407:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.022412:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022420:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.022428:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.022431:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.022435:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.022437:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23621, 23621] 00000080:00200000:3.0:1775764896.022526:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.022530:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.022558:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96755712 i_blocks 90112 00000080:00200000:3.0:1775764896.022562:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.022576:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96755712, count: 4096 00000080:00200000:3.0:1775764896.022585:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96755712, bytes: 4096 00000080:00200000:3.0:1775764896.022587:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.022591:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23623, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.022592:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96755712, 96759808) 0 0 00000080:00200000:3.0:1775764896.022595:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.022599:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23622, 23622] 00020000:00200000:3.0:1775764896.022602:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96755712 chunk: [96755712, 96759808) 96759808, 4096 00020000:00200000:3.0:1775764896.022605:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.022615:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23622, 23622] 00000008:00010000:3.0:1775764896.022620:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.022623:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.022630:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022639:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96755712 96759807) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022647:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022654:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022660:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.022666:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96755712, 96759808) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.022670:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23622 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.022686:0:18302:0:(rw26.c:858:ll_write_end()) pos 96755712, len 4096, copied 4096 00000080:00200000:3.0:1775764896.022689:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000061979992[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.022692:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000002a22c53 17ffffc0001001 3:0 ffff90c21280a0e0 23622 lru 00000080:00200000:3.0:1775764896.022697:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000f145376 23622: 1< 0 - - > 2< 96755712 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3858432 - | + - + - > 5< + - + - | 0 - | 3142 - -> 00000080:00200000:3.0:1775764896.022698:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000061979992 00000080:00200000:3.0:1775764896.022699:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.022701:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.022708:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.022715:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.022721:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.022732:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.022742:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.022744:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.022746:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.022755:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.022758:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.022760:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.022764:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.022771:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.022779:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.022781:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.022785:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.022788:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23622, 23622] 00000080:00200000:3.0:1775764896.022798:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.022801:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.022828:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96759808 i_blocks 90112 00000080:00200000:3.0:1775764896.022832:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.022847:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96759808, count: 4096 00000080:00200000:3.0:1775764896.022857:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96759808, bytes: 4096 00000080:00200000:3.0:1775764896.022859:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.022863:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23624, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.022865:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96759808, 96763904) 0 0 00000080:00200000:3.0:1775764896.022867:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.022974:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23623, 23623] 00020000:00200000:3.0:1775764896.022977:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96759808 chunk: [96759808, 96763904) 96763904, 4096 00020000:00200000:3.0:1775764896.022979:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.022993:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23623, 23623] 00000008:00010000:3.0:1775764896.022998:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.023001:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.023009:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023017:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96759808 96763903) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023026:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023033:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023039:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.023047:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96759808, 96763904) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.023052:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23623 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.023068:0:18302:0:(rw26.c:858:ll_write_end()) pos 96759808, len 4096, copied 4096 00000080:00200000:3.0:1775764896.023072:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000dc0dc344[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.023075:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003b2bea97 17ffffc0001001 3:0 ffff90c21280a1c0 23623 lru 00000080:00200000:3.0:1775764896.023081:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000040b4134a 23623: 1< 0 - - > 2< 96759808 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3854336 - | + - + - > 5< + - + - | 0 - | 3143 - -> 00000080:00200000:3.0:1775764896.023083:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000dc0dc344 00000080:00200000:3.0:1775764896.023084:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.023086:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.023094:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.023101:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.023108:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.023125:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.023136:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.023139:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.023141:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.023150:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.023153:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.023155:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.023159:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023167:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.023175:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.023178:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.023181:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.023184:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23623, 23623] 00000080:00200000:3.0:1775764896.023194:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.023197:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.023225:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96763904 i_blocks 90112 00000080:00200000:3.0:1775764896.023229:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.023245:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96763904, count: 4096 00000080:00200000:3.0:1775764896.023255:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96763904, bytes: 4096 00000080:00200000:3.0:1775764896.023258:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.023262:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23625, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.023263:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96763904, 96768000) 0 0 00000080:00200000:3.0:1775764896.023265:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.023270:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23624, 23624] 00020000:00200000:3.0:1775764896.023273:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96763904 chunk: [96763904, 96768000) 96768000, 4096 00020000:00200000:3.0:1775764896.023275:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.023287:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23624, 23624] 00000008:00010000:3.0:1775764896.023292:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.023294:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.023302:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023311:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96763904 96767999) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023319:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023327:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023339:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.023347:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96763904, 96768000) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.023352:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23624 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.023371:0:18302:0:(rw26.c:858:ll_write_end()) pos 96763904, len 4096, copied 4096 00000080:00200000:3.0:1775764896.023375:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003a48e336[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.023378:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003d9cdab0 17ffffc0001001 3:0 ffff90c21280a7e0 23624 lru 00000080:00200000:3.0:1775764896.023383:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000083f34b61 23624: 1< 0 - - > 2< 96763904 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3850240 - | + - + - > 5< + - + - | 0 - | 3144 - -> 00000080:00200000:3.0:1775764896.023386:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003a48e336 00000080:00200000:3.0:1775764896.023387:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.023389:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.023396:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.023405:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.023412:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.023419:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.023431:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.023434:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.023437:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.023532:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.023536:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.023538:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.023543:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023551:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.023560:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.023563:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.023567:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.023570:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23624, 23624] 00000080:00200000:3.0:1775764896.023581:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.023584:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.023615:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96768000 i_blocks 90112 00000080:00200000:3.0:1775764896.023618:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.023634:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96768000, count: 4096 00000080:00200000:3.0:1775764896.023644:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96768000, bytes: 4096 00000080:00200000:3.0:1775764896.023647:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.023650:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23626, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.023652:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96768000, 96772096) 0 0 00000080:00200000:3.0:1775764896.023654:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.023660:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23625, 23625] 00020000:00200000:3.0:1775764896.023663:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96768000 chunk: [96768000, 96772096) 96772096, 4096 00020000:00200000:3.0:1775764896.023665:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.023675:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23625, 23625] 00000008:00010000:3.0:1775764896.023680:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.023682:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.023691:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023700:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96768000 96772095) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023708:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023715:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023721:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.023728:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96768000, 96772096) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.023734:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23625 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.023757:0:18302:0:(rw26.c:858:ll_write_end()) pos 96768000, len 4096, copied 4096 00000080:00200000:3.0:1775764896.023761:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000995244d5[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.023764:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f24e37ca 17ffffc0001021 2:0 ffff90c21280a9a0 23625 lru 00000080:00200000:3.0:1775764896.023770:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000075406bb6 23625: 1< 0 - - > 2< 96768000 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3846144 - | + - + - > 5< + - + - | 0 - | 3145 - -> 00000080:00200000:3.0:1775764896.023772:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000995244d5 00000080:00200000:3.0:1775764896.023773:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.023775:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.023783:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.023791:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.023799:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.023806:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.023818:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.023821:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.023824:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.023833:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.023836:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.023839:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.023843:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.023851:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.023859:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.023861:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.023865:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.023868:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23625, 23625] 00000080:00200000:3.0:1775764896.024060:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.024064:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.024095:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96772096 i_blocks 90112 00000080:00200000:3.0:1775764896.024099:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.024128:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96772096, count: 4096 00000080:00200000:3.0:1775764896.024140:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96772096, bytes: 4096 00000080:00200000:3.0:1775764896.024143:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.024147:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23627, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.024148:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96772096, 96776192) 0 0 00000080:00200000:3.0:1775764896.024151:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.024157:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23626, 23626] 00020000:00200000:3.0:1775764896.024160:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96772096 chunk: [96772096, 96776192) 96776192, 4096 00020000:00200000:3.0:1775764896.024162:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.024175:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23626, 23626] 00000008:00010000:3.0:1775764896.024180:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.024183:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.024191:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024200:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96772096 96776191) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024208:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024215:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024222:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.024229:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96772096, 96776192) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.024234:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23626 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.024252:0:18302:0:(rw26.c:858:ll_write_end()) pos 96772096, len 4096, copied 4096 00000080:00200000:3.0:1775764896.024256:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001b0df07f[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.024259:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000154d69ac 17ffffc0001001 3:0 ffff90c21280a2a0 23626 lru 00000080:00200000:3.0:1775764896.024265:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ce19f9c5 23626: 1< 0 - - > 2< 96772096 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3842048 - | + - + - > 5< + - + - | 0 - | 3146 - -> 00000080:00200000:3.0:1775764896.024267:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001b0df07f 00000080:00200000:3.0:1775764896.024268:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.024270:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.024279:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.024287:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.024295:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.024302:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.024314:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.024317:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.024319:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.024329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.024331:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.024334:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.024338:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024346:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.024354:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.024357:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.024360:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.024363:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23626, 23626] 00000080:00200000:3.0:1775764896.024374:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.024377:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.024406:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96776192 i_blocks 90112 00000080:00200000:3.0:1775764896.024410:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.024425:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96776192, count: 4096 00000080:00200000:3.0:1775764896.024435:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96776192, bytes: 4096 00000080:00200000:3.0:1775764896.024438:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.024524:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23628, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.024525:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96776192, 96780288) 0 0 00000080:00200000:3.0:1775764896.024528:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.024534:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23627, 23627] 00020000:00200000:3.0:1775764896.024537:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96776192 chunk: [96776192, 96780288) 96780288, 4096 00020000:00200000:3.0:1775764896.024539:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.024553:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23627, 23627] 00000008:00010000:3.0:1775764896.024558:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.024561:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.024569:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024579:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96776192 96780287) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024587:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024600:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024606:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.024614:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96776192, 96780288) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.024619:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23627 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.024636:0:18302:0:(rw26.c:858:ll_write_end()) pos 96776192, len 4096, copied 4096 00000080:00200000:3.0:1775764896.024640:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001d83f349[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.024643:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006ac11bc3 17ffffc0001001 3:0 ffff90c21280ad20 23627 lru 00000080:00200000:3.0:1775764896.024649:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009c162d82 23627: 1< 0 - - > 2< 96776192 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3837952 - | + - + - > 5< + - + - | 0 - | 3147 - -> 00000080:00200000:3.0:1775764896.024651:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001d83f349 00000080:00200000:3.0:1775764896.024652:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.024654:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.024662:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.024669:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.024676:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.024683:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.024700:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.024703:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.024705:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.024716:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.024718:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.024721:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.024725:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024733:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.024741:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.024744:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.024748:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.024751:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23627, 23627] 00000080:00200000:3.0:1775764896.024762:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.024765:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.024796:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96780288 i_blocks 90112 00000080:00200000:3.0:1775764896.024800:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.024815:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96780288, count: 4096 00000080:00200000:3.0:1775764896.024825:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96780288, bytes: 4096 00000080:00200000:3.0:1775764896.024828:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.024832:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23629, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.024834:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96780288, 96784384) 0 0 00000080:00200000:3.0:1775764896.024836:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.024841:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23628, 23628] 00020000:00200000:3.0:1775764896.024844:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96780288 chunk: [96780288, 96784384) 96784384, 4096 00020000:00200000:3.0:1775764896.024846:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.024857:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23628, 23628] 00000008:00010000:3.0:1775764896.024861:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.024864:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.024890:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024977:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96780288 96784383) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024987:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.024994:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025001:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.025010:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96780288, 96784384) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.025016:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23628 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.025043:0:18302:0:(rw26.c:858:ll_write_end()) pos 96780288, len 4096, copied 4096 00000080:00200000:3.0:1775764896.025047:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cd30458b[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.025051:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001ad70b1d 17ffffc0001001 3:0 ffff90c21280b380 23628 lru 00000080:00200000:3.0:1775764896.025056:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000755b733e 23628: 1< 0 - - > 2< 96780288 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3833856 - | + - + - > 5< + - + - | 0 - | 3148 - -> 00000080:00200000:3.0:1775764896.025058:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cd30458b 00000080:00200000:3.0:1775764896.025060:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.025062:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.025071:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.025079:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.025086:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.025093:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.025106:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.025109:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.025111:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.025133:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.025136:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.025139:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.025143:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025152:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.025159:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.025162:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.025166:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.025169:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23628, 23628] 00000080:00200000:3.0:1775764896.025181:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.025185:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.025214:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96784384 i_blocks 90112 00000080:00200000:3.0:1775764896.025218:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.025234:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96784384, count: 4096 00000080:00200000:3.0:1775764896.025244:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96784384, bytes: 4096 00000080:00200000:3.0:1775764896.025246:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.025250:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23630, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.025252:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96784384, 96788480) 0 0 00000080:00200000:3.0:1775764896.025254:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.025260:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23629, 23629] 00020000:00200000:3.0:1775764896.025263:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96784384 chunk: [96784384, 96788480) 96788480, 4096 00020000:00200000:3.0:1775764896.025265:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.025277:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23629, 23629] 00000008:00010000:3.0:1775764896.025282:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.025285:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.025293:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025302:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96784384 96788479) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025311:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025319:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025326:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.025334:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96784384, 96788480) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.025339:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23629 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.025355:0:18302:0:(rw26.c:858:ll_write_end()) pos 96784384, len 4096, copied 4096 00000080:00200000:3.0:1775764896.025359:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006a388142[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.025362:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a3a67880 17ffffc0001001 3:0 ffff90c21280b620 23629 lru 00000080:00200000:3.0:1775764896.025367:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008ddfe510 23629: 1< 0 - - > 2< 96784384 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3829760 - | + - + - > 5< + - + - | 0 - | 3149 - -> 00000080:00200000:3.0:1775764896.025369:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006a388142 00000080:00200000:3.0:1775764896.025371:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.025373:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.025380:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.025388:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.025396:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.025403:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.025415:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.025418:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.025421:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.025430:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.025433:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.025435:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.025518:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025527:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.025537:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.025539:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.025544:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.025547:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23629, 23629] 00000080:00200000:3.0:1775764896.025557:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.025561:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.025590:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96788480 i_blocks 90112 00000080:00200000:3.0:1775764896.025594:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.025611:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96788480, count: 4096 00000080:00200000:3.0:1775764896.025622:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96788480, bytes: 4096 00000080:00200000:3.0:1775764896.025625:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.025629:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23631, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.025630:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96788480, 96792576) 0 0 00000080:00200000:3.0:1775764896.025633:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.025638:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23630, 23630] 00020000:00200000:3.0:1775764896.025641:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96788480 chunk: [96788480, 96792576) 96792576, 4096 00020000:00200000:3.0:1775764896.025643:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.025655:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23630, 23630] 00000008:00010000:3.0:1775764896.025661:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.025663:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.025670:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025679:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96788480 96792575) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025689:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025697:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025703:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.025711:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96788480, 96792576) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.025716:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23630 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.025738:0:18302:0:(rw26.c:858:ll_write_end()) pos 96788480, len 4096, copied 4096 00000080:00200000:3.0:1775764896.025742:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b7940616[3 00000000b99c1f5d 2 1 000000004d47a2a4] 00000080:00200000:3.0:1775764896.025745:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007a3b109e 17ffffc0001001 3:0 ffff90c21280be00 23630 lru 00000080:00200000:3.0:1775764896.025751:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a87db02a 23630: 1< 0 - - > 2< 96788480 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3825664 - | + - + - > 5< + - + - | 0 - | 3150 - -> 00000080:00200000:3.0:1775764896.025753:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b7940616 00000080:00200000:3.0:1775764896.025754:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.025756:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.025764:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.025771:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000f8e25b24 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.025778:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.025791:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.025802:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.025805:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.025807:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.025817:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.025819:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.025822:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.025826:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.025834:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.025843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6147b8 (ffff90c203cb9488/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.025845:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9488/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.025849:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.025851:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23630, 23630] 00000080:00200000:3.0:1775764896.025861:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.025864:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.033196:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96870400 i_blocks 90112 00000080:00200000:3.0:1775764896.033201:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.033226:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96870400, count: 4096 00000080:00200000:3.0:1775764896.033243:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96870400, bytes: 4096 00000080:00200000:3.0:1775764896.033246:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.033251:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23651, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.033253:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96870400, 96874496) 0 0 00000080:00200000:3.0:1775764896.033255:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.033261:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23650, 23650] 00020000:00200000:3.0:1775764896.033264:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96870400 chunk: [96870400, 96874496) 96874496, 4096 00020000:00200000:3.0:1775764896.033267:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.033286:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23650, 23650] 00000008:00010000:3.0:1775764896.033294:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.033296:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.033305:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033315:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96870400 96874495) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033323:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033331:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033338:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.033346:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96870400, 96874496) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.033351:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23650 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.033370:0:18302:0:(rw26.c:858:ll_write_end()) pos 96870400, len 4096, copied 4096 00000080:00200000:3.0:1775764896.033374:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d17724ce[3 00000000b99c1f5d 2 1 000000003b348dd4] 00000080:00200000:3.0:1775764896.033377:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000028b44aa0 17ffffc0001001 3:0 ffff90c21280b540 23650 lru 00000080:00200000:3.0:1775764896.033382:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ca6e1ca3 23650: 1< 0 - - > 2< 96870400 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3743744 - | + - + - > 5< + - + - | 0 - | 3170 - -> 00000080:00200000:3.0:1775764896.033385:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d17724ce 00000080:00200000:3.0:1775764896.033386:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.033388:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.033398:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c3025bd4 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.033406:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c3025bd4 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.033413:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.033421:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.033434:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.033436:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.033518:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.033534:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.033537:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.033540:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9bd8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.033544:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033552:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.033561:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.033563:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.033568:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.033571:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23650, 23650] 00000080:00200000:3.0:1775764896.033587:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.033591:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.033625:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96874496 i_blocks 90112 00000080:00200000:3.0:1775764896.033628:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.033647:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96874496, count: 4096 00000080:00200000:3.0:1775764896.033659:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96874496, bytes: 4096 00000080:00200000:3.0:1775764896.033662:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.033666:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23652, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.033667:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96874496, 96878592) 0 0 00000080:00200000:3.0:1775764896.033670:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.033675:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23651, 23651] 00020000:00200000:3.0:1775764896.033678:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96874496 chunk: [96874496, 96878592) 96878592, 4096 00020000:00200000:3.0:1775764896.033680:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.033692:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23651, 23651] 00000008:00010000:3.0:1775764896.033697:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.033699:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.033707:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033715:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96874496 96878591) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033724:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033730:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033743:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.033752:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96874496, 96878592) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.033757:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23651 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.033772:0:18302:0:(rw26.c:858:ll_write_end()) pos 96874496, len 4096, copied 4096 00000080:00200000:3.0:1775764896.033776:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000100fb6ea[3 00000000b99c1f5d 2 1 000000003b348dd4] 00000080:00200000:3.0:1775764896.033779:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004bffddb3 17ffffc0001001 3:0 ffff90c21280b460 23651 lru 00000080:00200000:3.0:1775764896.033786:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000025ebc880 23651: 1< 0 - - > 2< 96874496 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3739648 - | + - + - > 5< + - + - | 0 - | 3171 - -> 00000080:00200000:3.0:1775764896.033788:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000100fb6ea 00000080:00200000:3.0:1775764896.033789:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.033791:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.033799:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c3025bd4 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.033806:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c3025bd4 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.033812:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.033820:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.033831:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.033834:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.033836:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.033847:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.033850:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.033852:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9bd8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.033857:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.033865:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.033968:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.033972:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.033977:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.033981:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23651, 23651] 00000080:00200000:3.0:1775764896.033991:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.033996:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.034028:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96878592 i_blocks 90112 00000080:00200000:3.0:1775764896.034031:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.034049:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96878592, count: 4096 00000080:00200000:3.0:1775764896.034060:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96878592, bytes: 4096 00000080:00200000:3.0:1775764896.034063:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.034067:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23653, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.034068:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96878592, 96882688) 0 0 00000080:00200000:3.0:1775764896.034070:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.034076:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23652, 23652] 00020000:00200000:3.0:1775764896.034078:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96878592 chunk: [96878592, 96882688) 96882688, 4096 00020000:00200000:3.0:1775764896.034080:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.034093:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23652, 23652] 00000008:00010000:3.0:1775764896.034098:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.034101:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.034110:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034131:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96878592 96882687) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034140:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034147:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034154:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.034162:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96878592, 96882688) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.034167:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23652 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.034187:0:18302:0:(rw26.c:858:ll_write_end()) pos 96878592, len 4096, copied 4096 00000080:00200000:3.0:1775764896.034191:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d7dc813e[3 00000000b99c1f5d 2 1 000000003b348dd4] 00000080:00200000:3.0:1775764896.034194:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b15de27b 17ffffc0001001 3:0 ffff90c21280ba80 23652 lru 00000080:00200000:3.0:1775764896.034200:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cb5ff493 23652: 1< 0 - - > 2< 96878592 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3735552 - | + - + - > 5< + - + - | 0 - | 3172 - -> 00000080:00200000:3.0:1775764896.034203:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d7dc813e 00000080:00200000:3.0:1775764896.034204:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.034207:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.034216:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c3025bd4 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.034223:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c3025bd4 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.034231:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.034239:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.034251:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.034254:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.034256:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.034267:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.034270:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.034272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9bd8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.034276:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034284:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.034293:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.034295:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.034299:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.034303:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23652, 23652] 00000080:00200000:3.0:1775764896.034314:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.034317:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.034347:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96882688 i_blocks 90112 00000080:00200000:3.0:1775764896.034351:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.034367:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96882688, count: 4096 00000080:00200000:3.0:1775764896.034378:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96882688, bytes: 4096 00000080:00200000:3.0:1775764896.034381:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.034384:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23654, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.034386:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96882688, 96886784) 0 0 00000080:00200000:3.0:1775764896.034388:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.034394:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23653, 23653] 00020000:00200000:3.0:1775764896.034397:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96882688 chunk: [96882688, 96886784) 96886784, 4096 00020000:00200000:3.0:1775764896.034400:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.034411:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23653, 23653] 00000008:00010000:3.0:1775764896.034416:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.034419:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.034427:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034435:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96882688 96886783) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034523:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034531:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034538:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.034547:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96882688, 96886784) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.034554:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23653 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.034572:0:18302:0:(rw26.c:858:ll_write_end()) pos 96882688, len 4096, copied 4096 00000080:00200000:3.0:1775764896.034577:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005c78b112[3 00000000b99c1f5d 2 1 000000003b348dd4] 00000080:00200000:3.0:1775764896.034580:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007336e22f 17ffffc0001001 3:0 ffff90c21280b700 23653 lru 00000080:00200000:3.0:1775764896.034586:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000070658cc3 23653: 1< 0 - - > 2< 96882688 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3731456 - | + - + - > 5< + - + - | 0 - | 3173 - -> 00000080:00200000:3.0:1775764896.034588:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005c78b112 00000080:00200000:3.0:1775764896.034588:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.034591:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.034598:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c3025bd4 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.034607:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c3025bd4 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.034613:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.034621:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.034633:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.034636:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.034638:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.034649:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.034651:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.034654:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eab0 (ffff90c203cb9bd8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.034658:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034665:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.034674:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6146c8 (ffff90c203cb9e18/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.034677:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eab0 (ffff90c203cb9e18/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.034681:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.034684:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23653, 23653] 00000080:00200000:3.0:1775764896.034695:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.034698:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.034730:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96886784 i_blocks 90112 00000080:00200000:3.0:1775764896.034734:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.034752:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96886784, count: 4096 00000080:00200000:3.0:1775764896.034763:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96886784, bytes: 4096 00000080:00200000:3.0:1775764896.034766:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.034769:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23655, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.034771:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96886784, 96890880) 0 0 00000080:00200000:3.0:1775764896.034773:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.034779:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23654, 23654] 00020000:00200000:3.0:1775764896.034782:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96886784 chunk: [96886784, 96890880) 96890880, 4096 00020000:00200000:3.0:1775764896.034784:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.034796:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23654, 23654] 00000008:00010000:3.0:1775764896.034801:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.034804:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eab0, osc lock ffff90c2126d71d0, flags 0x0 00010000:00010000:3.0:1775764896.034812:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034821:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96886784 96890879) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034829:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034841:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.034849:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.034857:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96886784, 96890880) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.034862:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23654 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.035061:0:18302:0:(rw26.c:858:ll_write_end()) pos 96886784, len 4096, copied 4096 00000080:00200000:3.0:1775764896.035066:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000de02ec98[3 00000000b99c1f5d 2 1 000000003b348dd4] 00000080:00200000:3.0:1775764896.035069:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005a3ff645 17ffffc0001001 3:0 ffff90c21280bc40 23654 lru 00000080:00200000:3.0:1775764896.035075:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c9bebd9e 23654: 1< 0 - - > 2< 96886784 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3727360 - | + - + - > 5< + - + - | 0 - | 3174 - -> 00000080:00200000:3.0:1775764896.035077:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000de02ec98 00000080:00200000:3.0:1775764896.035078:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.040219:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96940032 i_blocks 90112 00000080:00200000:3.0:1775764896.040223:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.040245:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96940032, count: 4096 00000080:00200000:3.0:1775764896.040265:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96940032, bytes: 4096 00000080:00200000:3.0:1775764896.040268:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.040271:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23668, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.040273:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96940032, 96944128) 0 0 00000080:00200000:3.0:1775764896.040275:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.040282:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23667, 23667] 00020000:00200000:3.0:1775764896.040285:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96940032 chunk: [96940032, 96944128) 96944128, 4096 00020000:00200000:3.0:1775764896.040287:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.040303:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23667, 23667] 00000008:00010000:3.0:1775764896.040309:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.040312:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.040322:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.040331:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96940032 96944127) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.040340:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.040347:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.040354:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.040361:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96940032, 96944128) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.040368:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23667 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.040390:0:18302:0:(rw26.c:858:ll_write_end()) pos 96940032, len 4096, copied 4096 00000080:00200000:3.0:1775764896.040395:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000034c72eed[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.040398:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000682d376d 17ffffc0001001 3:0 ffff90c21280b8c0 23667 lru 00000080:00200000:3.0:1775764896.040405:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000da128a9 23667: 1< 0 - - > 2< 96940032 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3674112 - | + - + - > 5< + - + - | 0 - | 3187 - -> 00000080:00200000:3.0:1775764896.040407:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000034c72eed 00000080:00200000:3.0:1775764896.040408:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.040411:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.040419:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.040427:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.040436:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.040533:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.040549:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.040552:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.040554:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.040568:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.040571:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.040574:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.040578:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.040586:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.040594:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.040596:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.040601:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.040604:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23667, 23667] 00000080:00200000:3.0:1775764896.040619:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.040623:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.040656:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96944128 i_blocks 90112 00000080:00200000:3.0:1775764896.040659:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.040680:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96944128, count: 4096 00000080:00200000:3.0:1775764896.040691:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96944128, bytes: 4096 00000080:00200000:3.0:1775764896.040694:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.040697:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23669, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.040699:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96944128, 96948224) 0 0 00000080:00200000:3.0:1775764896.040701:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.040707:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23668, 23668] 00020000:00200000:3.0:1775764896.040710:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96944128 chunk: [96944128, 96948224) 96948224, 4096 00020000:00200000:3.0:1775764896.040713:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.040724:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23668, 23668] 00000008:00010000:3.0:1775764896.040730:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.040732:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.040740:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.040748:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96944128 96948223) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.040756:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.040764:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.040772:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.040781:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96944128, 96948224) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.040786:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23668 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.040805:0:18302:0:(rw26.c:858:ll_write_end()) pos 96944128, len 4096, copied 4096 00000080:00200000:3.0:1775764896.040810:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002195f0bd[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.040813:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e9cef58b 17ffffc0001001 3:0 ffff90c21280bb60 23668 lru 00000080:00200000:3.0:1775764896.040820:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000067ac84fd 23668: 1< 0 - - > 2< 96944128 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3670016 - | + - + - > 5< + - + - | 0 - | 3188 - -> 00000080:00200000:3.0:1775764896.040822:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002195f0bd 00000080:00200000:3.0:1775764896.040823:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.040825:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.040833:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.040841:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.040849:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.040857:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.041013:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.041018:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.041021:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.041033:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.041036:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.041038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.041043:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041051:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.041060:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.041062:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.041067:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.041069:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23668, 23668] 00000080:00200000:3.0:1775764896.041081:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.041084:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.041128:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96948224 i_blocks 90112 00000080:00200000:3.0:1775764896.041132:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.041150:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96948224, count: 4096 00000080:00200000:3.0:1775764896.041162:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96948224, bytes: 4096 00000080:00200000:3.0:1775764896.041165:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.041168:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23670, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.041170:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96948224, 96952320) 0 0 00000080:00200000:3.0:1775764896.041173:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.041178:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23669, 23669] 00020000:00200000:3.0:1775764896.041181:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96948224 chunk: [96948224, 96952320) 96952320, 4096 00020000:00200000:3.0:1775764896.041184:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.041195:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23669, 23669] 00000008:00010000:3.0:1775764896.041200:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.041203:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.041211:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041220:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96948224 96952319) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041229:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041241:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041249:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.041256:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96948224, 96952320) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.041261:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23669 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.041280:0:18302:0:(rw26.c:858:ll_write_end()) pos 96948224, len 4096, copied 4096 00000080:00200000:3.0:1775764896.041284:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002636de3d[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.041288:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fbbc6050 17ffffc0001001 3:0 ffff90c21280b000 23669 lru 00000080:00200000:3.0:1775764896.041294:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000084913737 23669: 1< 0 - - > 2< 96948224 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3665920 - | + - + - > 5< + - + - | 0 - | 3189 - -> 00000080:00200000:3.0:1775764896.041296:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002636de3d 00000080:00200000:3.0:1775764896.041297:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.041299:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.041308:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.041315:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.041323:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.041330:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.041347:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.041350:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.041353:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.041362:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.041364:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.041367:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.041371:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041379:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.041387:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.041389:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.041393:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.041396:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23669, 23669] 00000080:00200000:3.0:1775764896.041406:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.041410:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.041523:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96952320 i_blocks 90112 00000080:00200000:3.0:1775764896.041528:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.041547:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96952320, count: 4096 00000080:00200000:3.0:1775764896.041559:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96952320, bytes: 4096 00000080:00200000:3.0:1775764896.041562:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.041565:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23671, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.041567:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96952320, 96956416) 0 0 00000080:00200000:3.0:1775764896.041569:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.041575:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23670, 23670] 00020000:00200000:3.0:1775764896.041578:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96952320 chunk: [96952320, 96956416) 96956416, 4096 00020000:00200000:3.0:1775764896.041580:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.041592:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23670, 23670] 00000008:00010000:3.0:1775764896.041598:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.041601:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.041610:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041618:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96952320 96956415) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041627:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041634:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041641:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.041649:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96952320, 96956416) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.041654:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23670 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.041675:0:18302:0:(rw26.c:858:ll_write_end()) pos 96952320, len 4096, copied 4096 00000080:00200000:3.0:1775764896.041679:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007665d9eb[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.041681:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000084973556 17ffffc0001001 3:0 ffff90c21280bee0 23670 lru 00000080:00200000:3.0:1775764896.041687:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000027d08c43 23670: 1< 0 - - > 2< 96952320 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3661824 - | + - + - > 5< + - + - | 0 - | 3190 - -> 00000080:00200000:3.0:1775764896.041689:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007665d9eb 00000080:00200000:3.0:1775764896.041690:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.041692:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.041702:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.041709:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.041715:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.041723:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.041734:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.041737:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.041739:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.041750:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.041753:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.041756:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.041760:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041768:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.041776:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.041778:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.041782:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.041785:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23670, 23670] 00000080:00200000:3.0:1775764896.041797:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.041801:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.041829:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96956416 i_blocks 90112 00000080:00200000:3.0:1775764896.041832:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.041849:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96956416, count: 4096 00000080:00200000:3.0:1775764896.041860:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96956416, bytes: 4096 00000080:00200000:3.0:1775764896.041863:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.041866:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23672, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.041891:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96956416, 96960512) 0 0 00000080:00200000:3.0:1775764896.041894:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.041900:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23671, 23671] 00020000:00200000:3.0:1775764896.041903:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96956416 chunk: [96956416, 96960512) 96960512, 4096 00020000:00200000:3.0:1775764896.041905:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.041918:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23671, 23671] 00000008:00010000:3.0:1775764896.041923:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.041925:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.041933:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041942:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96956416 96960511) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041950:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041958:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.041966:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.041975:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96956416, 96960512) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.041980:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23671 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.042004:0:18302:0:(rw26.c:858:ll_write_end()) pos 96956416, len 4096, copied 4096 00000080:00200000:3.0:1775764896.042008:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000271f585e[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.042011:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000093237f8f 17ffffc0001021 2:0 ffff90c21280b0e0 23671 lru 00000080:00200000:3.0:1775764896.042017:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000142a2233 23671: 1< 0 - - > 2< 96956416 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3657728 - | + - + - > 5< + - + - | 0 - | 3191 - -> 00000080:00200000:3.0:1775764896.042019:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000271f585e 00000080:00200000:3.0:1775764896.042020:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.042022:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.042030:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.042038:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.042045:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.042052:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.042064:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.042067:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.042069:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.042079:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.042081:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.042084:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.042088:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042096:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.042104:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.042106:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.042110:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.042113:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23671, 23671] 00000080:00200000:3.0:1775764896.042136:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.042139:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.042169:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96960512 i_blocks 90112 00000080:00200000:3.0:1775764896.042172:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.042189:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96960512, count: 4096 00000080:00200000:3.0:1775764896.042199:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96960512, bytes: 4096 00000080:00200000:3.0:1775764896.042201:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.042205:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23673, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.042206:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96960512, 96964608) 0 0 00000080:00200000:3.0:1775764896.042209:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.042214:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23672, 23672] 00020000:00200000:3.0:1775764896.042218:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96960512 chunk: [96960512, 96964608) 96964608, 4096 00020000:00200000:3.0:1775764896.042220:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.042231:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23672, 23672] 00000008:00010000:3.0:1775764896.042236:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.042239:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.042246:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042256:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96960512 96964607) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042265:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042273:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042280:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.042287:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96960512, 96964608) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.042292:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23672 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.042308:0:18302:0:(rw26.c:858:ll_write_end()) pos 96960512, len 4096, copied 4096 00000080:00200000:3.0:1775764896.042312:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000dff98708[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.042314:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000087024053 17ffffc0001001 3:0 ffff90c21280b1c0 23672 lru 00000080:00200000:3.0:1775764896.042320:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000dcbeb083 23672: 1< 0 - - > 2< 96960512 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3653632 - | + - + - > 5< + - + - | 0 - | 3192 - -> 00000080:00200000:3.0:1775764896.042322:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000dff98708 00000080:00200000:3.0:1775764896.042323:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.042325:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.042333:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.042341:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.042351:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.042363:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.042374:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.042377:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.042379:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.042389:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.042392:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.042394:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.042399:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042407:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.042415:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.042417:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.042421:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.042424:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23672, 23672] 00000080:00200000:3.0:1775764896.042435:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.042521:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.042554:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96964608 i_blocks 90112 00000080:00200000:3.0:1775764896.042557:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.042573:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96964608, count: 4096 00000080:00200000:3.0:1775764896.042584:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96964608, bytes: 4096 00000080:00200000:3.0:1775764896.042586:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.042591:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23674, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.042592:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96964608, 96968704) 0 0 00000080:00200000:3.0:1775764896.042594:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.042600:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23673, 23673] 00020000:00200000:3.0:1775764896.042603:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96964608 chunk: [96964608, 96968704) 96968704, 4096 00020000:00200000:3.0:1775764896.042605:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.042617:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23673, 23673] 00000008:00010000:3.0:1775764896.042622:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.042625:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.042634:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042643:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96964608 96968703) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042652:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042659:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042667:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.042676:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96964608, 96968704) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.042682:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23673 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.042700:0:18302:0:(rw26.c:858:ll_write_end()) pos 96964608, len 4096, copied 4096 00000080:00200000:3.0:1775764896.042705:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000089db6756[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.042708:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003b09580c 17ffffc0001001 3:0 ffff90c21280b7e0 23673 lru 00000080:00200000:3.0:1775764896.042713:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f28d7a81 23673: 1< 0 - - > 2< 96964608 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3649536 - | + - + - > 5< + - + - | 0 - | 3193 - -> 00000080:00200000:3.0:1775764896.042716:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000089db6756 00000080:00200000:3.0:1775764896.042717:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.042719:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.042726:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.042734:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.042741:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.042748:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.042760:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.042763:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.042766:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.042775:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.042778:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.042780:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.042784:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.042791:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.042799:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.042802:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.042806:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.042809:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23673, 23673] 00000080:00200000:3.0:1775764896.042819:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.042823:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.042853:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96968704 i_blocks 90112 00000080:00200000:3.0:1775764896.042856:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.042968:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96968704, count: 4096 00000080:00200000:3.0:1775764896.042981:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96968704, bytes: 4096 00000080:00200000:3.0:1775764896.042984:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.042987:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23675, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.042989:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96968704, 96972800) 0 0 00000080:00200000:3.0:1775764896.042991:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.042997:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23674, 23674] 00020000:00200000:3.0:1775764896.043000:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96968704 chunk: [96968704, 96972800) 96972800, 4096 00020000:00200000:3.0:1775764896.043002:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.043013:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23674, 23674] 00000008:00010000:3.0:1775764896.043018:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.043021:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.043029:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043037:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96968704 96972799) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043046:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043054:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043066:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.043074:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96968704, 96972800) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.043080:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23674 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.043097:0:18302:0:(rw26.c:858:ll_write_end()) pos 96968704, len 4096, copied 4096 00000080:00200000:3.0:1775764896.043101:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004b7fec73[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.043103:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000019f68239 17ffffc0001001 3:0 ffff90c21280b9a0 23674 lru 00000080:00200000:3.0:1775764896.043108:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e521e2ab 23674: 1< 0 - - > 2< 96968704 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3645440 - | + - + - > 5< + - + - | 0 - | 3194 - -> 00000080:00200000:3.0:1775764896.043110:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004b7fec73 00000080:00200000:3.0:1775764896.043111:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.043113:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.043132:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.043140:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.043147:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.043154:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.043165:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.043168:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.043171:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.043182:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.043184:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.043187:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.043191:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043198:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.043206:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.043208:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.043212:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.043215:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23674, 23674] 00000080:00200000:3.0:1775764896.043227:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.043230:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.043262:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96972800 i_blocks 90112 00000080:00200000:3.0:1775764896.043266:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.043282:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96972800, count: 4096 00000080:00200000:3.0:1775764896.043293:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96972800, bytes: 4096 00000080:00200000:3.0:1775764896.043296:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.043300:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23676, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.043302:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96972800, 96976896) 0 0 00000080:00200000:3.0:1775764896.043304:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.043309:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23675, 23675] 00020000:00200000:3.0:1775764896.043312:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96972800 chunk: [96972800, 96976896) 96976896, 4096 00020000:00200000:3.0:1775764896.043315:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.043326:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23675, 23675] 00000008:00010000:3.0:1775764896.043331:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.043334:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.043342:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043350:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96972800 96976895) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043358:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043364:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043371:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.043378:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96972800, 96976896) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.043383:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23675 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.043399:0:18302:0:(rw26.c:858:ll_write_end()) pos 96972800, len 4096, copied 4096 00000080:00200000:3.0:1775764896.043404:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d7f0b977[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.043407:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000249ecbb0 17ffffc0001001 3:0 ffff90c21280b2a0 23675 lru 00000080:00200000:3.0:1775764896.043412:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001c3bbd72 23675: 1< 0 - - > 2< 96972800 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3641344 - | + - + - > 5< + - + - | 0 - | 3195 - -> 00000080:00200000:3.0:1775764896.043414:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d7f0b977 00000080:00200000:3.0:1775764896.043415:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.043418:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.043426:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.043434:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.043534:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.043542:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.043556:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.043559:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.043562:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.043572:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.043575:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.043578:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.043582:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043590:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.043600:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.043602:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.043607:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.043610:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23675, 23675] 00000080:00200000:3.0:1775764896.043621:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.043623:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.043653:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96976896 i_blocks 90112 00000080:00200000:3.0:1775764896.043657:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.043674:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96976896, count: 4096 00000080:00200000:3.0:1775764896.043684:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96976896, bytes: 4096 00000080:00200000:3.0:1775764896.043687:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.043691:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23677, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.043692:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96976896, 96980992) 0 0 00000080:00200000:3.0:1775764896.043695:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.043700:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23676, 23676] 00020000:00200000:3.0:1775764896.043703:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96976896 chunk: [96976896, 96980992) 96980992, 4096 00020000:00200000:3.0:1775764896.043705:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.043717:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23676, 23676] 00000008:00010000:3.0:1775764896.043722:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.043725:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.043732:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043740:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96976896 96980991) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043748:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043755:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043761:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.043769:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96976896, 96980992) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.043774:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23676 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.043792:0:18302:0:(rw26.c:858:ll_write_end()) pos 96976896, len 4096, copied 4096 00000080:00200000:3.0:1775764896.043796:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009fd08ee5[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.043799:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ff4840b4 17ffffc0001001 3:0 ffff90c21280bd20 23676 lru 00000080:00200000:3.0:1775764896.043805:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e67f933f 23676: 1< 0 - - > 2< 96976896 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3637248 - | + - + - > 5< + - + - | 0 - | 3196 - -> 00000080:00200000:3.0:1775764896.043807:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009fd08ee5 00000080:00200000:3.0:1775764896.043809:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.043811:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.043819:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.043826:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.043833:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.043841:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.043851:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.043854:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.043856:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.043866:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.043890:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.043893:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.043897:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.043905:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.043915:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.043917:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.043921:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.043923:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23676, 23676] 00000080:00200000:3.0:1775764896.043934:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.043937:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.043966:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96980992 i_blocks 90112 00000080:00200000:3.0:1775764896.043969:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.043986:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96980992, count: 4096 00000080:00200000:3.0:1775764896.043996:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96980992, bytes: 4096 00000080:00200000:3.0:1775764896.043999:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.044003:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23678, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.044005:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96980992, 96985088) 0 0 00000080:00200000:3.0:1775764896.044007:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.044012:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23677, 23677] 00020000:00200000:3.0:1775764896.044015:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96980992 chunk: [96980992, 96985088) 96985088, 4096 00020000:00200000:3.0:1775764896.044017:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.044028:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23677, 23677] 00000008:00010000:3.0:1775764896.044034:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.044036:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.044044:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044052:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96980992 96985087) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044060:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044072:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044078:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.044085:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96980992, 96985088) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.044090:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23677 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.044129:0:18302:0:(rw26.c:858:ll_write_end()) pos 96980992, len 4096, copied 4096 00000080:00200000:3.0:1775764896.044133:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f23f3f46[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.044136:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000097d3aa69 17ffffc0001001 3:0 ffff90c21280ca80 23677 lru 00000080:00200000:3.0:1775764896.044142:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003ae27460 23677: 1< 0 - - > 2< 96980992 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3633152 - | + - + - > 5< + - + - | 0 - | 3197 - -> 00000080:00200000:3.0:1775764896.044144:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f23f3f46 00000080:00200000:3.0:1775764896.044145:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.044147:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.044155:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.044163:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.044171:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.044178:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.044195:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.044198:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.044201:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.044211:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.044214:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.044216:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.044220:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044228:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.044237:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.044239:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.044243:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.044246:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23677, 23677] 00000080:00200000:3.0:1775764896.044257:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.044260:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.044290:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96985088 i_blocks 90112 00000080:00200000:3.0:1775764896.044294:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.044312:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96985088, count: 4096 00000080:00200000:3.0:1775764896.044322:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96985088, bytes: 4096 00000080:00200000:3.0:1775764896.044325:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.044329:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23679, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.044330:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96985088, 96989184) 0 0 00000080:00200000:3.0:1775764896.044332:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.044338:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23678, 23678] 00020000:00200000:3.0:1775764896.044341:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96985088 chunk: [96985088, 96989184) 96989184, 4096 00020000:00200000:3.0:1775764896.044343:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.044355:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23678, 23678] 00000008:00010000:3.0:1775764896.044360:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.044362:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.044370:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044378:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96985088 96989183) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044388:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044396:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044403:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.044411:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96985088, 96989184) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.044416:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23678 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.044431:0:18302:0:(rw26.c:858:ll_write_end()) pos 96985088, len 4096, copied 4096 00000080:00200000:3.0:1775764896.044435:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c7664431[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.044438:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003d38bd54 17ffffc0001001 3:0 ffff90c21280c700 23678 lru 00000080:00200000:3.0:1775764896.044524:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e75b1a72 23678: 1< 0 - - > 2< 96985088 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3629056 - | + - + - > 5< + - + - | 0 - | 3198 - -> 00000080:00200000:3.0:1775764896.044527:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c7664431 00000080:00200000:3.0:1775764896.044528:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.044530:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.044540:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.044547:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.044554:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.044561:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.044573:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.044576:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.044578:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.044589:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.044591:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.044594:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.044598:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044606:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.044614:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.044617:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.044621:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.044623:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23678, 23678] 00000080:00200000:3.0:1775764896.044635:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.044638:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.044668:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 96989184 i_blocks 90112 00000080:00200000:3.0:1775764896.044671:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.044687:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 96989184, count: 4096 00000080:00200000:3.0:1775764896.044697:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 96989184, bytes: 4096 00000080:00200000:3.0:1775764896.044700:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.044703:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23680, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.044705:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [96989184, 96993280) 0 0 00000080:00200000:3.0:1775764896.044707:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.044712:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23679, 23679] 00020000:00200000:3.0:1775764896.044715:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 96989184 chunk: [96989184, 96993280) 96993280, 4096 00020000:00200000:3.0:1775764896.044717:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.044729:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23679, 23679] 00000008:00010000:3.0:1775764896.044734:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.044737:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.044745:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044754:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (96989184 96993279) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044763:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044771:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.044778:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.044786:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [96989184, 96993280) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.044791:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23679 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.044812:0:18302:0:(rw26.c:858:ll_write_end()) pos 96989184, len 4096, copied 4096 00000080:00200000:3.0:1775764896.044816:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006a9a2037[3 00000000b99c1f5d 2 1 00000000e07ad2c0] 00000080:00200000:3.0:1775764896.044819:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000004764dd3 17ffffc0001001 3:0 ffff90c21280cc40 23679 lru 00000080:00200000:3.0:1775764896.044825:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003685e38f 23679: 1< 0 - - > 2< 96989184 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3624960 - | + - + - > 5< + - + - | 0 - | 3199 - -> 00000080:00200000:3.0:1775764896.044827:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006a9a2037 00000080:00200000:3.0:1775764896.044828:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.044830:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.044838:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.044846:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000007d0cdefa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.044853:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.044861:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.044975:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.044979:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.044981:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.044991:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.044994:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.044996:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.045001:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.045008:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.045016:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614b78 (ffff90c203cb9908/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.045018:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9908/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.045023:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.045025:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23679, 23679] 00000080:00200000:3.0:1775764896.045036:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.045039:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.086571:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97390592 i_blocks 90112 00000080:00200000:3.0:1775764896.086579:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.086621:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97390592, count: 4096 00000080:00200000:3.0:1775764896.086641:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97390592, bytes: 4096 00000080:00200000:3.0:1775764896.086646:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.086653:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23778, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.086655:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97390592, 97394688) 0 0 00000080:00200000:3.0:1775764896.086659:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.086671:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23777, 23777] 00020000:00200000:3.0:1775764896.086677:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97390592 chunk: [97390592, 97394688) 97394688, 4096 00020000:00200000:3.0:1775764896.086681:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.086704:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23777, 23777] 00000008:00010000:3.0:1775764896.086716:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.086721:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.086742:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.086758:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97390592 97394687) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.086772:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.086782:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.086789:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.086800:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97390592, 97394688) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.086811:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23777 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.086855:0:18302:0:(rw26.c:858:ll_write_end()) pos 97390592, len 4096, copied 4096 00000080:00200000:3.0:1775764896.086862:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007a5d2d5a[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.086865:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000058254b99 17ffffc0001001 3:0 ffff90c21280c8c0 23777 lru 00000080:00200000:3.0:1775764896.086958:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000065a1699c 23777: 1< 0 - - > 2< 97390592 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3223552 - | + - + - > 5< + - + - | 0 - | 3297 - -> 00000080:00200000:3.0:1775764896.086961:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007a5d2d5a 00000080:00200000:3.0:1775764896.086962:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.086967:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.086987:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.086997:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.087003:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.087016:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.087036:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.087041:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.087044:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.087065:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.087068:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.087071:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.087076:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087085:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.087098:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.087100:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.087105:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.087109:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23777, 23777] 00000080:00200000:3.0:1775764896.087135:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.087138:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.087171:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97394688 i_blocks 90112 00000080:00200000:3.0:1775764896.087175:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.087192:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97394688, count: 4096 00000080:00200000:3.0:1775764896.087202:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97394688, bytes: 4096 00000080:00200000:3.0:1775764896.087204:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.087209:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23779, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.087210:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97394688, 97398784) 0 0 00000080:00200000:3.0:1775764896.087213:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.087219:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23778, 23778] 00020000:00200000:3.0:1775764896.087221:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97394688 chunk: [97394688, 97398784) 97398784, 4096 00020000:00200000:3.0:1775764896.087224:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.087235:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23778, 23778] 00000008:00010000:3.0:1775764896.087240:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.087242:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.087250:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087259:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97394688 97398783) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087268:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087276:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087282:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.087290:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97394688, 97398784) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.087294:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23778 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.087310:0:18302:0:(rw26.c:858:ll_write_end()) pos 97394688, len 4096, copied 4096 00000080:00200000:3.0:1775764896.087314:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000095510401[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.087316:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000044178fa3 17ffffc0001001 3:0 ffff90c21280cb60 23778 lru 00000080:00200000:3.0:1775764896.087322:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f47c7003 23778: 1< 0 - - > 2< 97394688 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3219456 - | + - + - > 5< + - + - | 0 - | 3298 - -> 00000080:00200000:3.0:1775764896.087325:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000095510401 00000080:00200000:3.0:1775764896.087326:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.087328:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.087335:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.087343:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.087350:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.087356:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.087368:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.087371:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.087373:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.087383:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.087386:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.087388:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.087392:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087401:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.087409:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.087412:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.087416:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.087419:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23778, 23778] 00000080:00200000:3.0:1775764896.087429:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.087432:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.087535:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97398784 i_blocks 90112 00000080:00200000:3.0:1775764896.087539:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.087555:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97398784, count: 4096 00000080:00200000:3.0:1775764896.087564:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97398784, bytes: 4096 00000080:00200000:3.0:1775764896.087566:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.087570:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23780, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.087572:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97398784, 97402880) 0 0 00000080:00200000:3.0:1775764896.087575:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.087580:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23779, 23779] 00020000:00200000:3.0:1775764896.087583:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97398784 chunk: [97398784, 97402880) 97402880, 4096 00020000:00200000:3.0:1775764896.087585:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.087595:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23779, 23779] 00000008:00010000:3.0:1775764896.087600:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.087602:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.087609:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087619:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97398784 97402879) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087627:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087635:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087648:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.087655:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97398784, 97402880) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.087660:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23779 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.087676:0:18302:0:(rw26.c:858:ll_write_end()) pos 97398784, len 4096, copied 4096 00000080:00200000:3.0:1775764896.087680:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006dd23162[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.087683:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000745811dc 17ffffc0001001 3:0 ffff90c21280c000 23779 lru 00000080:00200000:3.0:1775764896.087689:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009c4fb79c 23779: 1< 0 - - > 2< 97398784 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3215360 - | + - + - > 5< + - + - | 0 - | 3299 - -> 00000080:00200000:3.0:1775764896.087691:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006dd23162 00000080:00200000:3.0:1775764896.087692:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.087694:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.087701:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.087709:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.087716:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.087723:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.087734:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.087736:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.087739:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.087747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.087750:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.087752:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.087756:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087764:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.087772:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.087774:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.087777:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.087780:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23779, 23779] 00000080:00200000:3.0:1775764896.087789:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.087792:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.087816:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97402880 i_blocks 90112 00000080:00200000:3.0:1775764896.087820:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.087834:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97402880, count: 4096 00000080:00200000:3.0:1775764896.087843:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97402880, bytes: 4096 00000080:00200000:3.0:1775764896.087845:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.087848:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23781, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.087850:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97402880, 97406976) 0 0 00000080:00200000:3.0:1775764896.087852:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.087858:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23780, 23780] 00020000:00200000:3.0:1775764896.087860:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97402880 chunk: [97402880, 97406976) 97406976, 4096 00020000:00200000:3.0:1775764896.087863:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.087893:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23780, 23780] 00000008:00010000:3.0:1775764896.087898:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.087900:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.087907:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087916:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97402880 97406975) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087925:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087932:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.087939:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.087946:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97402880, 97406976) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.087951:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23780 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.087967:0:18302:0:(rw26.c:858:ll_write_end()) pos 97402880, len 4096, copied 4096 00000080:00200000:3.0:1775764896.087971:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c8652a82[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.087974:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d3be5ef2 17ffffc0001001 3:0 ffff90c21280cee0 23780 lru 00000080:00200000:3.0:1775764896.087979:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e335bb11 23780: 1< 0 - - > 2< 97402880 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3211264 - | + - + - > 5< + - + - | 0 - | 3300 - -> 00000080:00200000:3.0:1775764896.087981:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c8652a82 00000080:00200000:3.0:1775764896.087982:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.087985:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.087993:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088000:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088008:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088015:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.088025:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.088028:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.088030:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.088040:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.088042:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.088045:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.088049:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088056:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.088064:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.088066:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.088070:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.088073:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23780, 23780] 00000080:00200000:3.0:1775764896.088083:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.088086:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.088112:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97406976 i_blocks 90112 00000080:00200000:3.0:1775764896.088127:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.088143:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97406976, count: 4096 00000080:00200000:3.0:1775764896.088153:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97406976, bytes: 4096 00000080:00200000:3.0:1775764896.088155:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.088159:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23782, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.088160:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97406976, 97411072) 0 0 00000080:00200000:3.0:1775764896.088163:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.088168:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23781, 23781] 00020000:00200000:3.0:1775764896.088171:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97406976 chunk: [97406976, 97411072) 97411072, 4096 00020000:00200000:3.0:1775764896.088173:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.088183:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23781, 23781] 00000008:00010000:3.0:1775764896.088188:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.088190:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.088198:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088206:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97406976 97411071) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088214:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088221:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088228:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.088236:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97406976, 97411072) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.088241:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23781 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.088257:0:18302:0:(rw26.c:858:ll_write_end()) pos 97406976, len 4096, copied 4096 00000080:00200000:3.0:1775764896.088262:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000784a466f[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.088265:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003f870a2f 17ffffc0001001 3:0 ffff90c21280c0e0 23781 lru 00000080:00200000:3.0:1775764896.088271:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000bfa967b 23781: 1< 0 - - > 2< 97406976 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3207168 - | + - + - > 5< + - + - | 0 - | 3301 - -> 00000080:00200000:3.0:1775764896.088273:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000784a466f 00000080:00200000:3.0:1775764896.088274:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.088276:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.088283:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088290:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088297:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088305:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.088314:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.088317:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.088318:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.088327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.088329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.088331:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.088334:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088342:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.088351:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.088353:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.088357:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.088360:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23781, 23781] 00000080:00200000:3.0:1775764896.088370:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.088373:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.088402:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97411072 i_blocks 90112 00000080:00200000:3.0:1775764896.088406:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.088422:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97411072, count: 4096 00000080:00200000:3.0:1775764896.088432:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97411072, bytes: 4096 00000080:00200000:3.0:1775764896.088435:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.088514:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23783, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.088516:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97411072, 97415168) 0 0 00000080:00200000:3.0:1775764896.088519:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.088525:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23782, 23782] 00020000:00200000:3.0:1775764896.088528:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97411072 chunk: [97411072, 97415168) 97415168, 4096 00020000:00200000:3.0:1775764896.088531:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.088543:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23782, 23782] 00000008:00010000:3.0:1775764896.088547:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.088550:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.088557:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088566:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97411072 97415167) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088575:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088589:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088595:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.088602:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97411072, 97415168) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.088607:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23782 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.088623:0:18302:0:(rw26.c:858:ll_write_end()) pos 97411072, len 4096, copied 4096 00000080:00200000:3.0:1775764896.088627:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ee62656e[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.088630:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006131a7d1 17ffffc0001001 3:0 ffff90c21280c1c0 23782 lru 00000080:00200000:3.0:1775764896.088635:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002e982920 23782: 1< 0 - - > 2< 97411072 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3203072 - | + - + - > 5< + - + - | 0 - | 3302 - -> 00000080:00200000:3.0:1775764896.088637:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ee62656e 00000080:00200000:3.0:1775764896.088638:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.088640:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.088647:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088655:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088662:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088669:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.088684:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.088687:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.088689:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.088698:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.088700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.088703:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.088707:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088714:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.088722:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.088724:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.088728:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.088731:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23782, 23782] 00000080:00200000:3.0:1775764896.088740:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.088744:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.088768:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97415168 i_blocks 90112 00000080:00200000:3.0:1775764896.088771:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.088785:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97415168, count: 4096 00000080:00200000:3.0:1775764896.088794:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97415168, bytes: 4096 00000080:00200000:3.0:1775764896.088797:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.088800:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23784, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.088802:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97415168, 97419264) 0 0 00000080:00200000:3.0:1775764896.088804:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.088809:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23783, 23783] 00020000:00200000:3.0:1775764896.088812:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97415168 chunk: [97415168, 97419264) 97419264, 4096 00020000:00200000:3.0:1775764896.088815:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.088824:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23783, 23783] 00000008:00010000:3.0:1775764896.088828:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.088830:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.088838:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088846:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97415168 97419263) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088855:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088862:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.088890:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.088898:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97415168, 97419264) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.088903:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23783 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.088924:0:18302:0:(rw26.c:858:ll_write_end()) pos 97415168, len 4096, copied 4096 00000080:00200000:3.0:1775764896.088928:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000e653a29[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.088931:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000164d4593 17ffffc0001021 2:0 ffff90c21280c7e0 23783 lru 00000080:00200000:3.0:1775764896.088936:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d9da6179 23783: 1< 0 - - > 2< 97415168 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3198976 - | + - + - > 5< + - + - | 0 - | 3303 - -> 00000080:00200000:3.0:1775764896.088939:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000e653a29 00000080:00200000:3.0:1775764896.088940:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.088942:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.088950:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088958:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088965:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.088972:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.088982:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.088985:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.088988:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.088997:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.088999:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.089002:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.089006:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089014:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.089022:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.089024:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.089028:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.089030:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23783, 23783] 00000080:00200000:3.0:1775764896.089042:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.089044:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.089072:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97419264 i_blocks 90112 00000080:00200000:3.0:1775764896.089076:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.089090:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97419264, count: 4096 00000080:00200000:3.0:1775764896.089100:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97419264, bytes: 4096 00000080:00200000:3.0:1775764896.089102:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.089106:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23785, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.089107:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97419264, 97423360) 0 0 00000080:00200000:3.0:1775764896.089110:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.089126:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23784, 23784] 00020000:00200000:3.0:1775764896.089130:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97419264 chunk: [97419264, 97423360) 97423360, 4096 00020000:00200000:3.0:1775764896.089132:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.089143:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23784, 23784] 00000008:00010000:3.0:1775764896.089148:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.089151:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.089159:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089167:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97419264 97423359) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089174:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089181:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089188:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.089196:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97419264, 97423360) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.089201:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23784 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.089219:0:18302:0:(rw26.c:858:ll_write_end()) pos 97419264, len 4096, copied 4096 00000080:00200000:3.0:1775764896.089223:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003e24ab6c[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.089226:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ebd8b866 17ffffc0001001 3:0 ffff90c21280c9a0 23784 lru 00000080:00200000:3.0:1775764896.089230:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006e6923a5 23784: 1< 0 - - > 2< 97419264 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3194880 - | + - + - > 5< + - + - | 0 - | 3304 - -> 00000080:00200000:3.0:1775764896.089232:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003e24ab6c 00000080:00200000:3.0:1775764896.089233:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.089236:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.089244:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.089252:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.089259:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.089266:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.089277:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.089280:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.089282:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.089291:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.089294:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.089296:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.089300:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089307:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.089314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.089317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.089321:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.089324:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23784, 23784] 00000080:00200000:3.0:1775764896.089334:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.089338:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.089366:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97423360 i_blocks 90112 00000080:00200000:3.0:1775764896.089369:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.089385:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97423360, count: 4096 00000080:00200000:3.0:1775764896.089396:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97423360, bytes: 4096 00000080:00200000:3.0:1775764896.089399:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.089403:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23786, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.089405:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97423360, 97427456) 0 0 00000080:00200000:3.0:1775764896.089407:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.089412:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23785, 23785] 00020000:00200000:3.0:1775764896.089415:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97423360 chunk: [97423360, 97427456) 97427456, 4096 00020000:00200000:3.0:1775764896.089418:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.089429:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23785, 23785] 00000008:00010000:3.0:1775764896.089435:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.089437:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.089530:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089539:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97423360 97427455) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089548:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089556:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089563:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.089571:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97423360, 97427456) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.089575:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23785 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.089591:0:18302:0:(rw26.c:858:ll_write_end()) pos 97423360, len 4096, copied 4096 00000080:00200000:3.0:1775764896.089596:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000762f7623[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.089598:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000def5fef3 17ffffc0001001 3:0 ffff90c21280c2a0 23785 lru 00000080:00200000:3.0:1775764896.089604:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000234d6b0c 23785: 1< 0 - - > 2< 97423360 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3190784 - | + - + - > 5< + - + - | 0 - | 3305 - -> 00000080:00200000:3.0:1775764896.089606:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000762f7623 00000080:00200000:3.0:1775764896.089607:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.089609:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.089616:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.089624:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.089631:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.089645:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.089656:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.089658:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.089660:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.089669:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.089671:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.089674:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.089678:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089685:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.089693:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.089695:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.089699:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.089701:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23785, 23785] 00000080:00200000:3.0:1775764896.089712:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.089715:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.089740:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97427456 i_blocks 90112 00000080:00200000:3.0:1775764896.089743:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.089758:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97427456, count: 4096 00000080:00200000:3.0:1775764896.089767:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97427456, bytes: 4096 00000080:00200000:3.0:1775764896.089770:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.089774:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23787, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.089775:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97427456, 97431552) 0 0 00000080:00200000:3.0:1775764896.089778:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.089783:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23786, 23786] 00020000:00200000:3.0:1775764896.089786:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97427456 chunk: [97427456, 97431552) 97431552, 4096 00020000:00200000:3.0:1775764896.089788:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.089798:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23786, 23786] 00000008:00010000:3.0:1775764896.089802:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.089805:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.089812:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089821:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97427456 97431551) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089829:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089837:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089844:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.089852:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97427456, 97431552) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.089856:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23786 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.089892:0:18302:0:(rw26.c:858:ll_write_end()) pos 97427456, len 4096, copied 4096 00000080:00200000:3.0:1775764896.089896:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000dc3d4623[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.089899:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000009be3a56 17ffffc0001001 3:0 ffff90c21280cd20 23786 lru 00000080:00200000:3.0:1775764896.089905:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003688fd30 23786: 1< 0 - - > 2< 97427456 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3186688 - | + - + - > 5< + - + - | 0 - | 3306 - -> 00000080:00200000:3.0:1775764896.089907:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000dc3d4623 00000080:00200000:3.0:1775764896.089908:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.089910:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.089917:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.089925:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.089932:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.089939:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.089949:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.089952:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.089954:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.089963:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.089965:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.089968:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.089971:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.089980:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.089988:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.089990:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.089994:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.089997:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23786, 23786] 00000080:00200000:3.0:1775764896.090006:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.090009:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.090034:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97431552 i_blocks 90112 00000080:00200000:3.0:1775764896.090038:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.090052:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97431552, count: 4096 00000080:00200000:3.0:1775764896.090060:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97431552, bytes: 4096 00000080:00200000:3.0:1775764896.090063:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.090066:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23788, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.090068:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97431552, 97435648) 0 0 00000080:00200000:3.0:1775764896.090071:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.090076:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23787, 23787] 00020000:00200000:3.0:1775764896.090078:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97431552 chunk: [97431552, 97435648) 97435648, 4096 00020000:00200000:3.0:1775764896.090081:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.090090:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23787, 23787] 00000008:00010000:3.0:1775764896.090095:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.090097:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.090104:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090125:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97431552 97435647) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090134:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090142:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090154:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.090162:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97431552, 97435648) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.090167:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23787 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.090181:0:18302:0:(rw26.c:858:ll_write_end()) pos 97431552, len 4096, copied 4096 00000080:00200000:3.0:1775764896.090184:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009716307f[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.090187:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000070c6f506 17ffffc0001001 3:0 ffff90c21280c380 23787 lru 00000080:00200000:3.0:1775764896.090192:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000955f192a 23787: 1< 0 - - > 2< 97431552 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3182592 - | + - + - > 5< + - + - | 0 - | 3307 - -> 00000080:00200000:3.0:1775764896.090194:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009716307f 00000080:00200000:3.0:1775764896.090195:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.090197:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.090204:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.090212:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.090219:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.090226:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.090235:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.090238:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.090240:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.090248:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.090250:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.090253:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.090257:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090265:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.090272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.090274:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.090277:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.090280:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23787, 23787] 00000080:00200000:3.0:1775764896.090289:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.090292:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.090317:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97435648 i_blocks 90112 00000080:00200000:3.0:1775764896.090320:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.090334:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97435648, count: 4096 00000080:00200000:3.0:1775764896.090345:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97435648, bytes: 4096 00000080:00200000:3.0:1775764896.090347:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.090350:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23789, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.090352:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97435648, 97439744) 0 0 00000080:00200000:3.0:1775764896.090354:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.090359:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23788, 23788] 00020000:00200000:3.0:1775764896.090362:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97435648 chunk: [97435648, 97439744) 97439744, 4096 00020000:00200000:3.0:1775764896.090365:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.090374:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23788, 23788] 00000008:00010000:3.0:1775764896.090378:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.090381:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.090388:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090397:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97435648 97439743) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090406:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090414:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090420:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.090428:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97435648, 97439744) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.090432:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23788 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.090518:0:18302:0:(rw26.c:858:ll_write_end()) pos 97435648, len 4096, copied 4096 00000080:00200000:3.0:1775764896.090523:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000001ccc1a7[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.090526:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c9f1cecd 17ffffc0001001 3:0 ffff90c21280c620 23788 lru 00000080:00200000:3.0:1775764896.090532:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d0eb202b 23788: 1< 0 - - > 2< 97435648 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3178496 - | + - + - > 5< + - + - | 0 - | 3308 - -> 00000080:00200000:3.0:1775764896.090534:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000001ccc1a7 00000080:00200000:3.0:1775764896.090535:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.090537:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.090544:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.090552:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.090560:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.090568:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.090577:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.090580:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.090582:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.090591:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.090593:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.090595:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.090599:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090606:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.090614:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.090616:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.090619:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.090622:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23788, 23788] 00000080:00200000:3.0:1775764896.090631:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.090634:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.090660:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97439744 i_blocks 90112 00000080:00200000:3.0:1775764896.090663:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.090678:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97439744, count: 4096 00000080:00200000:3.0:1775764896.090687:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97439744, bytes: 4096 00000080:00200000:3.0:1775764896.090689:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.090693:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23790, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.090695:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97439744, 97443840) 0 0 00000080:00200000:3.0:1775764896.090697:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.090702:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23789, 23789] 00020000:00200000:3.0:1775764896.090704:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97439744 chunk: [97439744, 97443840) 97443840, 4096 00020000:00200000:3.0:1775764896.090707:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.090716:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23789, 23789] 00000008:00010000:3.0:1775764896.090720:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.090723:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.090730:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090739:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97439744 97443839) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090748:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090755:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090762:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.090770:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97439744, 97443840) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.090774:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23789 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.090788:0:18302:0:(rw26.c:858:ll_write_end()) pos 97439744, len 4096, copied 4096 00000080:00200000:3.0:1775764896.090792:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007c154ed2[3 00000000b99c1f5d 2 1 00000000aca8ce63] 00000080:00200000:3.0:1775764896.090795:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d5fac93d 17ffffc0001001 3:0 ffff90c21280ce00 23789 lru 00000080:00200000:3.0:1775764896.090801:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000043c9af18 23789: 1< 0 - - > 2< 97439744 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3174400 - | + - + - > 5< + - + - | 0 - | 3309 - -> 00000080:00200000:3.0:1775764896.090803:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007c154ed2 00000080:00200000:3.0:1775764896.090804:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.090806:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.090812:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.090820:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009ac9f30a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.090827:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.090834:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.090843:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.090845:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.090847:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.090854:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.090857:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.090859:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.090862:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.090971:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.090982:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614f38 (ffff90c203cb9ea8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.090984:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9ea8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.090988:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.090991:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23789, 23789] 00000080:00200000:3.0:1775764896.091001:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.091004:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.095183:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97492992 i_blocks 90112 00000080:00200000:1.0:1775764896.095187:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.095210:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97492992, count: 4096 00000080:00200000:1.0:1775764896.095228:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97492992, bytes: 4096 00000080:00200000:1.0:1775764896.095230:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.095234:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23803, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.095235:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97492992, 97497088) 0 0 00000080:00200000:1.0:1775764896.095238:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.095244:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23802, 23802] 00020000:00200000:1.0:1775764896.095247:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97492992 chunk: [97492992, 97497088) 97497088, 4096 00020000:00200000:1.0:1775764896.095249:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.095265:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23802, 23802] 00000008:00010000:1.0:1775764896.095271:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.095275:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.095284:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095293:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97492992 97497087) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095303:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095310:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095317:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.095325:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97492992, 97497088) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.095331:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23802 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.095362:0:18302:0:(rw26.c:858:ll_write_end()) pos 97492992, len 4096, copied 4096 00000080:00200000:1.0:1775764896.095366:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000dbb19cc8[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.095369:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000057ab63df 17ffffc0001001 3:0 ffff90c20b3e70e0 23802 lru 00000080:00200000:1.0:1775764896.095374:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000028caab1a 23802: 1< 0 - - > 2< 97492992 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3121152 - | + - + - > 5< + - + - | 0 - | 3322 - -> 00000080:00200000:1.0:1775764896.095376:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000dbb19cc8 00000080:00200000:1.0:1775764896.095377:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.095379:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.095388:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.095396:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.095404:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.095411:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.095424:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.095427:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.095429:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.095528:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.095531:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.095534:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.095539:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095547:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.095556:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.095559:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.095564:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.095566:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23802, 23802] 00000080:00200000:1.0:1775764896.095585:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.095589:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.095623:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97497088 i_blocks 90112 00000080:00200000:1.0:1775764896.095627:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.095645:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97497088, count: 4096 00000080:00200000:1.0:1775764896.095656:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97497088, bytes: 4096 00000080:00200000:1.0:1775764896.095659:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.095662:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23804, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.095664:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97497088, 97501184) 0 0 00000080:00200000:1.0:1775764896.095666:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.095671:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23803, 23803] 00020000:00200000:1.0:1775764896.095674:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97497088 chunk: [97497088, 97501184) 97501184, 4096 00020000:00200000:1.0:1775764896.095676:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.095688:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23803, 23803] 00000008:00010000:1.0:1775764896.095693:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.095696:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.095704:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095712:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97497088 97501183) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095721:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095733:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095739:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.095747:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97497088, 97501184) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.095752:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23803 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.095769:0:18302:0:(rw26.c:858:ll_write_end()) pos 97497088, len 4096, copied 4096 00000080:00200000:1.0:1775764896.095773:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000acda10f9[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.095776:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000022c718dd 17ffffc0001001 3:0 ffff90c20b3e71c0 23803 lru 00000080:00200000:1.0:1775764896.095781:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000099ca1be0 23803: 1< 0 - - > 2< 97497088 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3117056 - | + - + - > 5< + - + - | 0 - | 3323 - -> 00000080:00200000:1.0:1775764896.095783:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000acda10f9 00000080:00200000:1.0:1775764896.095784:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.095786:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.095794:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.095801:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.095807:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.095814:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.095830:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.095834:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.095836:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.095845:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.095848:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.095850:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.095853:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.095861:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.095891:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.095894:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.095898:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.095901:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23803, 23803] 00000080:00200000:1.0:1775764896.095912:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.095915:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.095945:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97501184 i_blocks 90112 00000080:00200000:1.0:1775764896.095949:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.095965:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97501184, count: 4096 00000080:00200000:1.0:1775764896.095976:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97501184, bytes: 4096 00000080:00200000:1.0:1775764896.095979:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.095982:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23805, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.095983:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97501184, 97505280) 0 0 00000080:00200000:1.0:1775764896.095985:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.095991:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23804, 23804] 00020000:00200000:1.0:1775764896.095994:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97501184 chunk: [97501184, 97505280) 97505280, 4096 00020000:00200000:1.0:1775764896.095996:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.096007:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23804, 23804] 00000008:00010000:1.0:1775764896.096013:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.096015:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.096023:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096032:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97501184 97505279) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096040:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096047:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096053:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.096061:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97501184, 97505280) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.096066:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23804 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.096082:0:18302:0:(rw26.c:858:ll_write_end()) pos 97501184, len 4096, copied 4096 00000080:00200000:1.0:1775764896.096087:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000025534631[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.096089:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008eb8c7a5 17ffffc0001001 3:0 ffff90c20b3e77e0 23804 lru 00000080:00200000:1.0:1775764896.096094:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000759188a9 23804: 1< 0 - - > 2< 97501184 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3112960 - | + - + - > 5< + - + - | 0 - | 3324 - -> 00000080:00200000:1.0:1775764896.096096:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000025534631 00000080:00200000:1.0:1775764896.096098:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.096100:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.096108:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.096133:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.096140:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.096147:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.096158:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.096161:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.096163:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.096174:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.096176:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.096179:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.096183:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096191:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.096199:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.096201:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.096205:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.096208:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23804, 23804] 00000080:00200000:1.0:1775764896.096220:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.096223:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.096251:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97505280 i_blocks 90112 00000080:00200000:1.0:1775764896.096255:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.096270:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97505280, count: 4096 00000080:00200000:1.0:1775764896.096281:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97505280, bytes: 4096 00000080:00200000:1.0:1775764896.096284:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.096288:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23806, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.096289:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97505280, 97509376) 0 0 00000080:00200000:1.0:1775764896.096291:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.096296:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23805, 23805] 00020000:00200000:1.0:1775764896.096299:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97505280 chunk: [97505280, 97509376) 97509376, 4096 00020000:00200000:1.0:1775764896.096301:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.096311:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23805, 23805] 00000008:00010000:1.0:1775764896.096316:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.096319:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.096326:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096335:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97505280 97509375) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096343:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096350:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096356:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.096364:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97505280, 97509376) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.096369:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23805 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.096385:0:18302:0:(rw26.c:858:ll_write_end()) pos 97505280, len 4096, copied 4096 00000080:00200000:1.0:1775764896.096390:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e1a234be[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.096393:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006454f410 17ffffc0001001 3:0 ffff90c20b3e79a0 23805 lru 00000080:00200000:1.0:1775764896.096399:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000500a9395 23805: 1< 0 - - > 2< 97505280 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3108864 - | + - + - > 5< + - + - | 0 - | 3325 - -> 00000080:00200000:1.0:1775764896.096401:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e1a234be 00000080:00200000:1.0:1775764896.096402:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.096404:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.096412:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.096420:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.096427:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.096435:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.096530:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.096534:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.096536:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.096546:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.096549:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.096552:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.096556:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096564:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.096573:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.096575:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.096579:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.096582:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23805, 23805] 00000080:00200000:1.0:1775764896.096593:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.096596:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.096626:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97509376 i_blocks 90112 00000080:00200000:1.0:1775764896.096629:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.096645:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97509376, count: 4096 00000080:00200000:1.0:1775764896.096656:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97509376, bytes: 4096 00000080:00200000:1.0:1775764896.096659:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.096663:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23807, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.096664:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97509376, 97513472) 0 0 00000080:00200000:1.0:1775764896.096666:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.096671:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23806, 23806] 00020000:00200000:1.0:1775764896.096675:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97509376 chunk: [97509376, 97513472) 97513472, 4096 00020000:00200000:1.0:1775764896.096677:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.096688:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23806, 23806] 00000008:00010000:1.0:1775764896.096694:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.096697:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.096705:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096713:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97509376 97513471) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096722:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096729:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096735:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.096743:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97509376, 97513472) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.096748:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23806 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.096766:0:18302:0:(rw26.c:858:ll_write_end()) pos 97509376, len 4096, copied 4096 00000080:00200000:1.0:1775764896.096770:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c95febc1[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.096773:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a00a2702 17ffffc0001001 3:0 ffff90c20b3e72a0 23806 lru 00000080:00200000:1.0:1775764896.096778:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000f1914fd 23806: 1< 0 - - > 2< 97509376 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3104768 - | + - + - > 5< + - + - | 0 - | 3326 - -> 00000080:00200000:1.0:1775764896.096781:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c95febc1 00000080:00200000:1.0:1775764896.096782:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.096784:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.096791:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.096799:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.096805:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.096818:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.096830:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.096833:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.096835:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.096846:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.096849:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.096851:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.096855:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.096863:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.096969:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.096972:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.096977:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.096980:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23806, 23806] 00000080:00200000:1.0:1775764896.096991:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.096995:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.097026:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97513472 i_blocks 90112 00000080:00200000:1.0:1775764896.097030:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.097046:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97513472, count: 4096 00000080:00200000:1.0:1775764896.097057:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97513472, bytes: 4096 00000080:00200000:1.0:1775764896.097060:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.097064:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23808, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.097065:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97513472, 97517568) 0 0 00000080:00200000:1.0:1775764896.097068:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.097073:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23807, 23807] 00020000:00200000:1.0:1775764896.097076:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97513472 chunk: [97513472, 97517568) 97517568, 4096 00020000:00200000:1.0:1775764896.097079:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.097091:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23807, 23807] 00000008:00010000:1.0:1775764896.097096:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.097099:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.097107:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097126:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97513472 97517567) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097134:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097142:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097149:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.097157:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97513472, 97517568) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.097161:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23807 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.097184:0:18302:0:(rw26.c:858:ll_write_end()) pos 97513472, len 4096, copied 4096 00000080:00200000:1.0:1775764896.097188:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000025d8242b[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.097190:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006aa1ef45 17ffffc0001021 2:0 ffff90c20b3e7d20 23807 lru 00000080:00200000:1.0:1775764896.097195:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003735a1a5 23807: 1< 0 - - > 2< 97513472 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3100672 - | + - + - > 5< + - + - | 0 - | 3327 - -> 00000080:00200000:1.0:1775764896.097198:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000025d8242b 00000080:00200000:1.0:1775764896.097199:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.097201:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.097208:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.097216:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.097223:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.097230:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.097241:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.097244:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.097246:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.097256:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.097259:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.097262:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.097266:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097274:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.097283:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.097285:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.097289:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.097292:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23807, 23807] 00000080:00200000:1.0:1775764896.097302:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.097304:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.097332:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97517568 i_blocks 90112 00000080:00200000:1.0:1775764896.097335:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.097350:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97517568, count: 4096 00000080:00200000:1.0:1775764896.097360:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97517568, bytes: 4096 00000080:00200000:1.0:1775764896.097363:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.097366:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23809, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.097368:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97517568, 97521664) 0 0 00000080:00200000:1.0:1775764896.097370:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.097375:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23808, 23808] 00020000:00200000:1.0:1775764896.097378:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97517568 chunk: [97517568, 97521664) 97521664, 4096 00020000:00200000:1.0:1775764896.097380:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.097391:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23808, 23808] 00000008:00010000:1.0:1775764896.097396:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.097398:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.097406:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097415:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97517568 97521663) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097423:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097431:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097525:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.097535:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97517568, 97521664) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.097541:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23808 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.097564:0:18302:0:(rw26.c:858:ll_write_end()) pos 97517568, len 4096, copied 4096 00000080:00200000:1.0:1775764896.097569:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000036319a88[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.097571:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a874e72b 17ffffc0001001 3:0 ffff90c20b3e7380 23808 lru 00000080:00200000:1.0:1775764896.097577:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001a92c526 23808: 1< 0 - - > 2< 97517568 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3096576 - | + - + - > 5< + - + - | 0 - | 3328 - -> 00000080:00200000:1.0:1775764896.097579:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000036319a88 00000080:00200000:1.0:1775764896.097580:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.097582:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.097590:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.097597:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.097603:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.097610:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.097622:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.097626:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.097629:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.097640:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.097642:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.097644:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.097648:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097656:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.097664:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.097667:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.097671:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.097674:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23808, 23808] 00000080:00200000:1.0:1775764896.097685:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.097687:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.097717:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97521664 i_blocks 90112 00000080:00200000:1.0:1775764896.097721:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.097738:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97521664, count: 4096 00000080:00200000:1.0:1775764896.097749:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97521664, bytes: 4096 00000080:00200000:1.0:1775764896.097752:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.097755:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23810, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.097757:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97521664, 97525760) 0 0 00000080:00200000:1.0:1775764896.097759:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.097764:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23809, 23809] 00020000:00200000:1.0:1775764896.097767:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97521664 chunk: [97521664, 97525760) 97525760, 4096 00020000:00200000:1.0:1775764896.097769:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.097782:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23809, 23809] 00000008:00010000:1.0:1775764896.097787:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.097789:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.097797:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097806:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97521664 97525759) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097814:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097821:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097828:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.097835:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97521664, 97525760) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.097841:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23809 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.097859:0:18302:0:(rw26.c:858:ll_write_end()) pos 97521664, len 4096, copied 4096 00000080:00200000:1.0:1775764896.097863:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a1d31bb3[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.097866:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000113777aa 17ffffc0001001 3:0 ffff90c20b3e7620 23809 lru 00000080:00200000:1.0:1775764896.097895:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000058c8d343 23809: 1< 0 - - > 2< 97521664 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3092480 - | + - + - > 5< + - + - | 0 - | 3329 - -> 00000080:00200000:1.0:1775764896.097897:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a1d31bb3 00000080:00200000:1.0:1775764896.097899:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.097901:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.097910:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.097918:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.097925:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.097933:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.097945:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.097949:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.097951:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.097962:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.097965:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.097967:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.097971:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.097979:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.097987:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.097989:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.097993:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.097996:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23809, 23809] 00000080:00200000:1.0:1775764896.098007:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.098010:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.098039:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97525760 i_blocks 90112 00000080:00200000:1.0:1775764896.098043:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.098060:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97525760, count: 4096 00000080:00200000:1.0:1775764896.098071:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97525760, bytes: 4096 00000080:00200000:1.0:1775764896.098074:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.098078:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23811, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.098079:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97525760, 97529856) 0 0 00000080:00200000:1.0:1775764896.098082:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.098087:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23810, 23810] 00020000:00200000:1.0:1775764896.098091:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97525760 chunk: [97525760, 97529856) 97529856, 4096 00020000:00200000:1.0:1775764896.098093:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.098105:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23810, 23810] 00000008:00010000:1.0:1775764896.098110:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.098112:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.098131:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098139:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97525760 97529855) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098147:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098154:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098160:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.098167:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97525760, 97529856) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.098172:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23810 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.098188:0:18302:0:(rw26.c:858:ll_write_end()) pos 97525760, len 4096, copied 4096 00000080:00200000:1.0:1775764896.098192:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000227cc6c7[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.098195:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000086e5ded1 17ffffc0001001 3:0 ffff90c20b3e7e00 23810 lru 00000080:00200000:1.0:1775764896.098201:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c0ebd744 23810: 1< 0 - - > 2< 97525760 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3088384 - | + - + - > 5< + - + - | 0 - | 3330 - -> 00000080:00200000:1.0:1775764896.098203:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000227cc6c7 00000080:00200000:1.0:1775764896.098204:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.098207:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.098215:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.098223:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.098230:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.098236:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.098247:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.098250:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.098252:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.098262:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.098264:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.098267:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.098271:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098278:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.098287:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.098289:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.098293:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.098296:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23810, 23810] 00000080:00200000:1.0:1775764896.098308:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.098311:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.098343:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97529856 i_blocks 90112 00000080:00200000:1.0:1775764896.098346:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.098363:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97529856, count: 4096 00000080:00200000:1.0:1775764896.098373:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97529856, bytes: 4096 00000080:00200000:1.0:1775764896.098376:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.098379:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23812, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.098381:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97529856, 97533952) 0 0 00000080:00200000:1.0:1775764896.098383:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.098388:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23811, 23811] 00020000:00200000:1.0:1775764896.098391:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97529856 chunk: [97529856, 97533952) 97533952, 4096 00020000:00200000:1.0:1775764896.098394:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.098405:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23811, 23811] 00000008:00010000:1.0:1775764896.098410:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.098413:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.098421:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098430:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97529856 97533951) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098525:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098540:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098547:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.098556:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97529856, 97533952) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.098561:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23811 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.098579:0:18302:0:(rw26.c:858:ll_write_end()) pos 97529856, len 4096, copied 4096 00000080:00200000:1.0:1775764896.098583:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e62b5856[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.098587:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e45440af 17ffffc0001001 3:0 ffff90c20b3e7540 23811 lru 00000080:00200000:1.0:1775764896.098592:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000406696f6 23811: 1< 0 - - > 2< 97529856 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3084288 - | + - + - > 5< + - + - | 0 - | 3331 - -> 00000080:00200000:1.0:1775764896.098594:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e62b5856 00000080:00200000:1.0:1775764896.098595:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.098597:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.098605:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.098613:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.098620:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.098628:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.098644:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.098647:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.098649:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.098659:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.098662:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.098664:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.098668:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098676:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.098685:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.098687:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.098691:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.098694:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23811, 23811] 00000080:00200000:1.0:1775764896.098705:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.098709:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.098739:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97533952 i_blocks 90112 00000080:00200000:1.0:1775764896.098743:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.098758:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97533952, count: 4096 00000080:00200000:1.0:1775764896.098769:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97533952, bytes: 4096 00000080:00200000:1.0:1775764896.098771:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.098775:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23813, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.098776:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97533952, 97538048) 0 0 00000080:00200000:1.0:1775764896.098779:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.098784:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23812, 23812] 00020000:00200000:1.0:1775764896.098788:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97533952 chunk: [97533952, 97538048) 97538048, 4096 00020000:00200000:1.0:1775764896.098791:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.098803:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23812, 23812] 00000008:00010000:1.0:1775764896.098809:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.098811:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.098819:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098827:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97533952 97538047) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098835:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098842:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098848:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.098856:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97533952, 97538048) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.098861:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23812 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.098899:0:18302:0:(rw26.c:858:ll_write_end()) pos 97533952, len 4096, copied 4096 00000080:00200000:1.0:1775764896.098904:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006c2e7549[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.098907:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007731e5d8 17ffffc0001001 3:0 ffff90c20b3e7460 23812 lru 00000080:00200000:1.0:1775764896.098913:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b76265b8 23812: 1< 0 - - > 2< 97533952 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3080192 - | + - + - > 5< + - + - | 0 - | 3332 - -> 00000080:00200000:1.0:1775764896.098915:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006c2e7549 00000080:00200000:1.0:1775764896.098916:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.098919:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.098926:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.098933:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.098941:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.098948:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.098960:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.098963:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.098965:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.098975:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.098977:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.098980:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.098984:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.098993:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.099000:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.099003:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.099007:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.099009:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23812, 23812] 00000080:00200000:1.0:1775764896.099020:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.099024:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.099055:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97538048 i_blocks 90112 00000080:00200000:1.0:1775764896.099059:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.099075:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97538048, count: 4096 00000080:00200000:1.0:1775764896.099086:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97538048, bytes: 4096 00000080:00200000:1.0:1775764896.099089:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.099092:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23814, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.099094:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97538048, 97542144) 0 0 00000080:00200000:1.0:1775764896.099096:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.099101:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23813, 23813] 00020000:00200000:1.0:1775764896.099104:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97538048 chunk: [97538048, 97542144) 97542144, 4096 00020000:00200000:1.0:1775764896.099106:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.099129:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23813, 23813] 00000008:00010000:1.0:1775764896.099134:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.099137:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.099144:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099154:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97538048 97542143) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099163:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099170:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099177:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.099186:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97538048, 97542144) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.099191:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23813 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.099207:0:18302:0:(rw26.c:858:ll_write_end()) pos 97538048, len 4096, copied 4096 00000080:00200000:1.0:1775764896.099211:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000004871b09[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.099214:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b1b0d87f 17ffffc0001001 3:0 ffff90c20b3e7a80 23813 lru 00000080:00200000:1.0:1775764896.099219:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002c944bf5 23813: 1< 0 - - > 2< 97538048 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3076096 - | + - + - > 5< + - + - | 0 - | 3333 - -> 00000080:00200000:1.0:1775764896.099221:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000004871b09 00000080:00200000:1.0:1775764896.099222:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.099224:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.099231:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.099238:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.099244:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.099251:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.099262:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.099265:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.099267:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.099276:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.099278:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.099281:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.099284:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099292:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.099299:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.099301:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.099305:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.099309:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23813, 23813] 00000080:00200000:1.0:1775764896.099320:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.099323:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.099353:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97542144 i_blocks 90112 00000080:00200000:1.0:1775764896.099357:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.099372:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97542144, count: 4096 00000080:00200000:1.0:1775764896.099382:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97542144, bytes: 4096 00000080:00200000:1.0:1775764896.099385:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.099389:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23815, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.099390:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97542144, 97546240) 0 0 00000080:00200000:1.0:1775764896.099393:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.099398:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23814, 23814] 00020000:00200000:1.0:1775764896.099401:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97542144 chunk: [97542144, 97546240) 97546240, 4096 00020000:00200000:1.0:1775764896.099403:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.099415:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23814, 23814] 00000008:00010000:1.0:1775764896.099420:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.099422:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.099430:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099528:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97542144 97546239) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099540:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099547:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099554:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.099563:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97542144, 97546240) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.099570:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23814 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.099588:0:18302:0:(rw26.c:858:ll_write_end()) pos 97542144, len 4096, copied 4096 00000080:00200000:1.0:1775764896.099592:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f70923c2[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.099595:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a3584ed6 17ffffc0001001 3:0 ffff90c20b3e7700 23814 lru 00000080:00200000:1.0:1775764896.099601:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c43a5030 23814: 1< 0 - - > 2< 97542144 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3072000 - | + - + - > 5< + - + - | 0 - | 3334 - -> 00000080:00200000:1.0:1775764896.099603:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f70923c2 00000080:00200000:1.0:1775764896.099604:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.099606:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.099614:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.099622:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.099628:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.099641:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.099653:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.099656:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.099658:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.099668:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.099670:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.099673:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.099677:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099684:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.099691:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.099693:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.099697:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.099701:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23814, 23814] 00000080:00200000:1.0:1775764896.099712:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.099715:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.099747:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97546240 i_blocks 90112 00000080:00200000:1.0:1775764896.099751:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.099767:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97546240, count: 4096 00000080:00200000:1.0:1775764896.099777:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97546240, bytes: 4096 00000080:00200000:1.0:1775764896.099780:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.099784:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23816, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.099785:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97546240, 97550336) 0 0 00000080:00200000:1.0:1775764896.099787:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.099793:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23815, 23815] 00020000:00200000:1.0:1775764896.099796:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97546240 chunk: [97546240, 97550336) 97550336, 4096 00020000:00200000:1.0:1775764896.099798:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.099810:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23815, 23815] 00000008:00010000:1.0:1775764896.099816:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.099819:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.099828:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099838:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97546240 97550335) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099847:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099855:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.099862:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.099894:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97546240, 97550336) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.099900:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23815 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.099919:0:18302:0:(rw26.c:858:ll_write_end()) pos 97546240, len 4096, copied 4096 00000080:00200000:1.0:1775764896.099924:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006a11fac9[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.099927:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000086c05f2a 17ffffc0001001 3:0 ffff90c20b3e7c40 23815 lru 00000080:00200000:1.0:1775764896.099932:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004f58413f 23815: 1< 0 - - > 2< 97546240 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3067904 - | + - + - > 5< + - + - | 0 - | 3335 - -> 00000080:00200000:1.0:1775764896.099934:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006a11fac9 00000080:00200000:1.0:1775764896.099935:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.099938:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.099945:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.099954:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.099961:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.099968:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.099979:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.099982:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.099984:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.099995:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.099998:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.100000:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.100003:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100011:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.100019:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.100023:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.100026:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.100029:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23815, 23815] 00000080:00200000:1.0:1775764896.100040:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.100043:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.100073:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97550336 i_blocks 90112 00000080:00200000:1.0:1775764896.100077:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.100092:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97550336, count: 4096 00000080:00200000:1.0:1775764896.100103:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97550336, bytes: 4096 00000080:00200000:1.0:1775764896.100105:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.100109:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23817, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.100110:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97550336, 97554432) 0 0 00000080:00200000:1.0:1775764896.100112:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.100129:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23816, 23816] 00020000:00200000:1.0:1775764896.100132:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97550336 chunk: [97550336, 97554432) 97554432, 4096 00020000:00200000:1.0:1775764896.100134:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.100145:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23816, 23816] 00000008:00010000:1.0:1775764896.100150:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.100152:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.100160:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100168:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97550336 97554431) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100177:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100184:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100196:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.100204:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97550336, 97554432) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.100210:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23816 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.100226:0:18302:0:(rw26.c:858:ll_write_end()) pos 97550336, len 4096, copied 4096 00000080:00200000:1.0:1775764896.100231:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000057081772[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.100234:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003486dc19 17ffffc0001001 3:0 ffff90c20b3e78c0 23816 lru 00000080:00200000:1.0:1775764896.100239:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009cd135b8 23816: 1< 0 - - > 2< 97550336 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3063808 - | + - + - > 5< + - + - | 0 - | 3336 - -> 00000080:00200000:1.0:1775764896.100241:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000057081772 00000080:00200000:1.0:1775764896.100242:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.100245:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.100253:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.100260:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.100268:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.100275:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.100285:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.100288:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.100290:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.100299:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.100302:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.100304:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.100308:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100315:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.100323:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.100326:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.100330:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.100333:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23816, 23816] 00000080:00200000:1.0:1775764896.100344:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.100347:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.100376:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97554432 i_blocks 90112 00000080:00200000:1.0:1775764896.100380:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.100395:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97554432, count: 4096 00000080:00200000:1.0:1775764896.100406:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97554432, bytes: 4096 00000080:00200000:1.0:1775764896.100409:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.100413:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23818, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.100414:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97554432, 97558528) 0 0 00000080:00200000:1.0:1775764896.100416:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.100422:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23817, 23817] 00020000:00200000:1.0:1775764896.100425:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97554432 chunk: [97554432, 97558528) 97558528, 4096 00020000:00200000:1.0:1775764896.100427:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.100523:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23817, 23817] 00000008:00010000:1.0:1775764896.100532:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.100535:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.100544:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100554:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97554432 97558527) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100562:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100570:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100577:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.100585:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97554432, 97558528) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.100591:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23817 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.100609:0:18302:0:(rw26.c:858:ll_write_end()) pos 97554432, len 4096, copied 4096 00000080:00200000:1.0:1775764896.100613:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000046c69dcb[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.100616:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e8908b0e 17ffffc0001001 3:0 ffff90c20b3e7b60 23817 lru 00000080:00200000:1.0:1775764896.100621:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000eaeb7c3f 23817: 1< 0 - - > 2< 97554432 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3059712 - | + - + - > 5< + - + - | 0 - | 3337 - -> 00000080:00200000:1.0:1775764896.100623:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000046c69dcb 00000080:00200000:1.0:1775764896.100624:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.100626:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.100633:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.100641:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.100648:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.100655:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.100667:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.100671:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.100673:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.100682:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.100685:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.100687:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.100692:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100700:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.100708:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.100711:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.100715:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.100718:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23817, 23817] 00000080:00200000:1.0:1775764896.100728:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.100732:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.100763:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97558528 i_blocks 90112 00000080:00200000:1.0:1775764896.100766:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.100783:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97558528, count: 4096 00000080:00200000:1.0:1775764896.100794:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97558528, bytes: 4096 00000080:00200000:1.0:1775764896.100797:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.100801:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23819, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.100802:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97558528, 97562624) 0 0 00000080:00200000:1.0:1775764896.100804:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.100809:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23818, 23818] 00020000:00200000:1.0:1775764896.100812:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97558528 chunk: [97558528, 97562624) 97562624, 4096 00020000:00200000:1.0:1775764896.100814:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.100826:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23818, 23818] 00000008:00010000:1.0:1775764896.100830:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.100833:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.100841:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100850:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97558528 97562623) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100859:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100866:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.100896:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.100904:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97558528, 97562624) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.100909:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23818 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.100927:0:18302:0:(rw26.c:858:ll_write_end()) pos 97558528, len 4096, copied 4096 00000080:00200000:1.0:1775764896.100931:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008d31daad[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.100933:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005b9bc034 17ffffc0001001 3:0 ffff90c20b3e7000 23818 lru 00000080:00200000:1.0:1775764896.100938:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000972b4848 23818: 1< 0 - - > 2< 97558528 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3055616 - | + - + - > 5< + - + - | 0 - | 3338 - -> 00000080:00200000:1.0:1775764896.100940:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008d31daad 00000080:00200000:1.0:1775764896.100941:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.100943:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.100951:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.100958:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.100965:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.100972:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.100984:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.100987:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.100989:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.100998:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.101001:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.101003:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.101007:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101015:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.101024:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.101027:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.101031:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.101033:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23818, 23818] 00000080:00200000:1.0:1775764896.101043:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.101046:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.101075:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97562624 i_blocks 90112 00000080:00200000:1.0:1775764896.101079:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.101095:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97562624, count: 4096 00000080:00200000:1.0:1775764896.101105:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97562624, bytes: 4096 00000080:00200000:1.0:1775764896.101107:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.101111:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23820, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.101112:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97562624, 97566720) 0 0 00000080:00200000:1.0:1775764896.101126:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.101131:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23819, 23819] 00020000:00200000:1.0:1775764896.101134:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97562624 chunk: [97562624, 97566720) 97566720, 4096 00020000:00200000:1.0:1775764896.101136:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.101147:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23819, 23819] 00000008:00010000:1.0:1775764896.101153:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.101155:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.101162:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101171:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97562624 97566719) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101179:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101191:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101197:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.101204:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97562624, 97566720) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.101210:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23819 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.101225:0:18302:0:(rw26.c:858:ll_write_end()) pos 97562624, len 4096, copied 4096 00000080:00200000:1.0:1775764896.101229:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e27cf5ce[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.101232:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f57e7946 17ffffc0001001 3:0 ffff90c20b3e7ee0 23819 lru 00000080:00200000:1.0:1775764896.101238:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000dcda6635 23819: 1< 0 - - > 2< 97562624 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3051520 - | + - + - > 5< + - + - | 0 - | 3339 - -> 00000080:00200000:1.0:1775764896.101240:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e27cf5ce 00000080:00200000:1.0:1775764896.101241:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.101243:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.101251:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.101258:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.101266:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.101273:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.101289:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.101292:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.101294:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.101305:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.101307:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.101309:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.101312:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101320:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.101327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.101330:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.101334:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.101336:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23819, 23819] 00000080:00200000:1.0:1775764896.101347:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.101350:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.101380:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97566720 i_blocks 90112 00000080:00200000:1.0:1775764896.101383:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.101399:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97566720, count: 4096 00000080:00200000:1.0:1775764896.101410:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97566720, bytes: 4096 00000080:00200000:1.0:1775764896.101412:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.101416:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23821, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.101418:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97566720, 97570816) 0 0 00000080:00200000:1.0:1775764896.101420:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.101424:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23820, 23820] 00020000:00200000:1.0:1775764896.101428:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97566720 chunk: [97566720, 97570816) 97570816, 4096 00020000:00200000:1.0:1775764896.101430:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.101527:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23820, 23820] 00000008:00010000:1.0:1775764896.101535:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.101537:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.101545:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101554:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97566720 97570815) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101563:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101570:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101576:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.101584:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97566720, 97570816) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.101590:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23820 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.101617:0:18302:0:(rw26.c:858:ll_write_end()) pos 97566720, len 4096, copied 4096 00000080:00200000:1.0:1775764896.101622:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000846a7df0[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.101625:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000be6834db 17ffffc0001001 3:0 ffff90c20b3e8620 23820 lru 00000080:00200000:1.0:1775764896.101631:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000870e79f4 23820: 1< 0 - - > 2< 97566720 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3047424 - | + - + - > 5< + - + - | 0 - | 3340 - -> 00000080:00200000:1.0:1775764896.101633:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000846a7df0 00000080:00200000:1.0:1775764896.101635:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.101637:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.101645:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.101652:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.101659:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.101667:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.101678:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.101681:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.101683:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.101692:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.101694:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.101697:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.101701:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101709:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.101717:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.101720:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.101725:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.101727:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23820, 23820] 00000080:00200000:1.0:1775764896.101738:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.101742:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.101770:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97570816 i_blocks 90112 00000080:00200000:1.0:1775764896.101774:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.101789:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97570816, count: 4096 00000080:00200000:1.0:1775764896.101800:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97570816, bytes: 4096 00000080:00200000:1.0:1775764896.101802:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.101806:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23822, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.101808:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97570816, 97574912) 0 0 00000080:00200000:1.0:1775764896.101810:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.101815:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23821, 23821] 00020000:00200000:1.0:1775764896.101818:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97570816 chunk: [97570816, 97574912) 97574912, 4096 00020000:00200000:1.0:1775764896.101820:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.101830:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23821, 23821] 00000008:00010000:1.0:1775764896.101834:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.101837:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.101844:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101852:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97570816 97574911) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101859:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101865:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.101895:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.101904:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97570816, 97574912) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.101909:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23821 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.101926:0:18302:0:(rw26.c:858:ll_write_end()) pos 97570816, len 4096, copied 4096 00000080:00200000:1.0:1775764896.101930:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000baec44ff[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.101933:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b2e44479 17ffffc0001001 3:0 ffff90c20b3e8e00 23821 lru 00000080:00200000:1.0:1775764896.101938:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c1b7a2ae 23821: 1< 0 - - > 2< 97570816 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3043328 - | + - + - > 5< + - + - | 0 - | 3341 - -> 00000080:00200000:1.0:1775764896.101939:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000baec44ff 00000080:00200000:1.0:1775764896.101941:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.101943:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.101950:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.101958:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.101965:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.101973:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.101984:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.101987:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.101989:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.101999:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.102001:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.102004:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.102008:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102016:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.102024:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.102026:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.102030:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.102033:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23821, 23821] 00000080:00200000:1.0:1775764896.102044:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.102047:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.102075:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97574912 i_blocks 90112 00000080:00200000:1.0:1775764896.102079:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.102096:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97574912, count: 4096 00000080:00200000:1.0:1775764896.102107:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97574912, bytes: 4096 00000080:00200000:1.0:1775764896.102110:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.102141:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23823, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.102143:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97574912, 97579008) 0 0 00000080:00200000:1.0:1775764896.102146:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.102152:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23822, 23822] 00020000:00200000:1.0:1775764896.102155:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97574912 chunk: [97574912, 97579008) 97579008, 4096 00020000:00200000:1.0:1775764896.102157:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.102169:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23822, 23822] 00000008:00010000:1.0:1775764896.102174:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.102177:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.102185:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102194:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97574912 97579007) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102203:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102210:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102218:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.102226:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97574912, 97579008) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.102232:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23822 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.102255:0:18302:0:(rw26.c:858:ll_write_end()) pos 97574912, len 4096, copied 4096 00000080:00200000:1.0:1775764896.102260:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cd964ebd[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.102263:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000004f14422 17ffffc0001021 2:0 ffff90c20b3e8540 23822 lru 00000080:00200000:1.0:1775764896.102268:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e217935c 23822: 1< 0 - - > 2< 97574912 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3039232 - | + - + - > 5< + - + - | 0 - | 3342 - -> 00000080:00200000:1.0:1775764896.102270:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cd964ebd 00000080:00200000:1.0:1775764896.102271:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.102274:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.102282:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.102290:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.102296:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.102309:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.102321:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.102324:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.102326:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.102336:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.102339:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.102341:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.102345:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102352:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.102359:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.102362:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.102366:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.102369:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23822, 23822] 00000080:00200000:1.0:1775764896.102380:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.102383:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.102413:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97579008 i_blocks 90112 00000080:00200000:1.0:1775764896.102417:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.102433:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97579008, count: 4096 00000080:00200000:1.0:1775764896.102542:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97579008, bytes: 4096 00000080:00200000:1.0:1775764896.102546:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.102551:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23824, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.102552:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97579008, 97583104) 0 0 00000080:00200000:1.0:1775764896.102554:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.102560:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23823, 23823] 00020000:00200000:1.0:1775764896.102563:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97579008 chunk: [97579008, 97583104) 97583104, 4096 00020000:00200000:1.0:1775764896.102565:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.102576:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23823, 23823] 00000008:00010000:1.0:1775764896.102582:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.102585:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.102593:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102603:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97579008 97583103) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102611:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102619:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102624:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.102632:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97579008, 97583104) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.102638:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23823 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.102653:0:18302:0:(rw26.c:858:ll_write_end()) pos 97579008, len 4096, copied 4096 00000080:00200000:1.0:1775764896.102657:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004946f9d6[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.102659:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006cf6fb43 17ffffc0001001 3:0 ffff90c20b3e8460 23823 lru 00000080:00200000:1.0:1775764896.102665:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000eda29ef2 23823: 1< 0 - - > 2< 97579008 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3035136 - | + - + - > 5< + - + - | 0 - | 3343 - -> 00000080:00200000:1.0:1775764896.102667:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004946f9d6 00000080:00200000:1.0:1775764896.102668:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.102671:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.102678:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.102686:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.102693:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.102701:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.102713:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.102716:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.102718:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.102728:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.102731:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.102733:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.102737:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102745:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.102753:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.102756:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.102760:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.102763:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23823, 23823] 00000080:00200000:1.0:1775764896.102774:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.102777:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.102807:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97583104 i_blocks 90112 00000080:00200000:1.0:1775764896.102810:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.102827:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97583104, count: 4096 00000080:00200000:1.0:1775764896.102837:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97583104, bytes: 4096 00000080:00200000:1.0:1775764896.102840:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.102844:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23825, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.102845:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97583104, 97587200) 0 0 00000080:00200000:1.0:1775764896.102847:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.102852:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23824, 23824] 00020000:00200000:1.0:1775764896.102855:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97583104 chunk: [97583104, 97587200) 97587200, 4096 00020000:00200000:1.0:1775764896.102857:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.102868:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23824, 23824] 00000008:00010000:1.0:1775764896.102897:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.102900:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.102907:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102917:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97583104 97587199) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102925:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102933:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.102945:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.102954:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97583104, 97587200) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.102960:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23824 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.102978:0:18302:0:(rw26.c:858:ll_write_end()) pos 97583104, len 4096, copied 4096 00000080:00200000:1.0:1775764896.102982:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000086120c44[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.102985:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007bd71251 17ffffc0001001 3:0 ffff90c20b3e8a80 23824 lru 00000080:00200000:1.0:1775764896.102990:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ec57031c 23824: 1< 0 - - > 2< 97583104 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3031040 - | + - + - > 5< + - + - | 0 - | 3344 - -> 00000080:00200000:1.0:1775764896.102992:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000086120c44 00000080:00200000:1.0:1775764896.102993:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.102995:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.103003:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.103010:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.103016:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.103023:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.103034:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.103036:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.103039:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.103048:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.103050:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.103053:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.103056:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103064:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.103071:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.103073:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.103077:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.103080:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23824, 23824] 00000080:00200000:1.0:1775764896.103091:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.103094:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.103136:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97587200 i_blocks 90112 00000080:00200000:1.0:1775764896.103140:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.103156:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97587200, count: 4096 00000080:00200000:1.0:1775764896.103167:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97587200, bytes: 4096 00000080:00200000:1.0:1775764896.103169:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.103173:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23826, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.103174:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97587200, 97591296) 0 0 00000080:00200000:1.0:1775764896.103177:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.103182:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23825, 23825] 00020000:00200000:1.0:1775764896.103185:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97587200 chunk: [97587200, 97591296) 97591296, 4096 00020000:00200000:1.0:1775764896.103188:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.103200:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23825, 23825] 00000008:00010000:1.0:1775764896.103205:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.103208:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.103216:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103225:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97587200 97591295) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103234:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103241:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103248:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.103257:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97587200, 97591296) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.103262:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23825 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.103277:0:18302:0:(rw26.c:858:ll_write_end()) pos 97587200, len 4096, copied 4096 00000080:00200000:1.0:1775764896.103281:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000bf444c86[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.103284:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d4412580 17ffffc0001001 3:0 ffff90c20b3e8700 23825 lru 00000080:00200000:1.0:1775764896.103289:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000b39d9c0 23825: 1< 0 - - > 2< 97587200 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3026944 - | + - + - > 5< + - + - | 0 - | 3345 - -> 00000080:00200000:1.0:1775764896.103291:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000bf444c86 00000080:00200000:1.0:1775764896.103292:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.103294:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.103302:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.103310:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.103318:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.103324:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.103335:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.103338:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.103340:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.103350:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.103352:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.103354:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.103358:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103365:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.103372:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.103375:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.103379:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.103381:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23825, 23825] 00000080:00200000:1.0:1775764896.103393:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.103396:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.103427:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97591296 i_blocks 90112 00000080:00200000:1.0:1775764896.103431:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.103535:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97591296, count: 4096 00000080:00200000:1.0:1775764896.103549:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97591296, bytes: 4096 00000080:00200000:1.0:1775764896.103552:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.103556:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23827, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.103558:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97591296, 97595392) 0 0 00000080:00200000:1.0:1775764896.103560:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.103566:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23826, 23826] 00020000:00200000:1.0:1775764896.103569:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97591296 chunk: [97591296, 97595392) 97595392, 4096 00020000:00200000:1.0:1775764896.103572:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.103585:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23826, 23826] 00000008:00010000:1.0:1775764896.103590:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.103593:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.103602:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103611:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97591296 97595391) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103621:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103630:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103637:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.103646:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97591296, 97595392) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.103651:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23826 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.103669:0:18302:0:(rw26.c:858:ll_write_end()) pos 97591296, len 4096, copied 4096 00000080:00200000:1.0:1775764896.103674:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d1c23d1f[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.103676:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000576bce0b 17ffffc0001001 3:0 ffff90c20b3e8c40 23826 lru 00000080:00200000:1.0:1775764896.103682:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005d5d9b80 23826: 1< 0 - - > 2< 97591296 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3022848 - | + - + - > 5< + - + - | 0 - | 3346 - -> 00000080:00200000:1.0:1775764896.103684:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d1c23d1f 00000080:00200000:1.0:1775764896.103685:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.103687:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.103695:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.103702:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.103709:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.103715:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.103727:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.103730:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.103732:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.103743:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.103745:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.103748:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.103751:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103759:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.103767:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.103769:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.103773:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.103775:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23826, 23826] 00000080:00200000:1.0:1775764896.103786:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.103789:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.103819:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97595392 i_blocks 90112 00000080:00200000:1.0:1775764896.103822:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.103836:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97595392, count: 4096 00000080:00200000:1.0:1775764896.103847:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97595392, bytes: 4096 00000080:00200000:1.0:1775764896.103849:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.103853:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23828, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.103855:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97595392, 97599488) 0 0 00000080:00200000:1.0:1775764896.103857:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.103862:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23827, 23827] 00020000:00200000:1.0:1775764896.103866:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97595392 chunk: [97595392, 97599488) 97599488, 4096 00020000:00200000:1.0:1775764896.103868:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.103906:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23827, 23827] 00000008:00010000:1.0:1775764896.103912:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.103914:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.103921:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103929:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97595392 97599487) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103938:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103951:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.103959:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.103967:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97595392, 97599488) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.103972:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23827 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.103989:0:18302:0:(rw26.c:858:ll_write_end()) pos 97595392, len 4096, copied 4096 00000080:00200000:1.0:1775764896.103994:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000acd97a99[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.103997:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000902a71c7 17ffffc0001001 3:0 ffff90c20b3e88c0 23827 lru 00000080:00200000:1.0:1775764896.104002:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e20b015f 23827: 1< 0 - - > 2< 97595392 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3018752 - | + - + - > 5< + - + - | 0 - | 3347 - -> 00000080:00200000:1.0:1775764896.104004:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000acd97a99 00000080:00200000:1.0:1775764896.104005:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.104007:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.104015:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.104023:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.104030:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.104038:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.104054:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.104057:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.104060:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.104070:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.104073:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.104075:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.104079:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104087:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.104095:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.104097:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.104101:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.104104:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23827, 23827] 00000080:00200000:1.0:1775764896.104144:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.104149:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.104181:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97599488 i_blocks 90112 00000080:00200000:1.0:1775764896.104185:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.104202:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97599488, count: 4096 00000080:00200000:1.0:1775764896.104213:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97599488, bytes: 4096 00000080:00200000:1.0:1775764896.104216:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.104220:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23829, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.104222:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97599488, 97603584) 0 0 00000080:00200000:1.0:1775764896.104224:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.104230:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23828, 23828] 00020000:00200000:1.0:1775764896.104233:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97599488 chunk: [97599488, 97603584) 97603584, 4096 00020000:00200000:1.0:1775764896.104236:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.104247:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23828, 23828] 00000008:00010000:1.0:1775764896.104252:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.104255:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.104262:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104272:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97599488 97603583) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104281:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104289:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104295:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.104304:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97599488, 97603584) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.104309:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23828 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.104327:0:18302:0:(rw26.c:858:ll_write_end()) pos 97599488, len 4096, copied 4096 00000080:00200000:1.0:1775764896.104331:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000014762fad[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.104334:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000044171088 17ffffc0001001 3:0 ffff90c20b3e8b60 23828 lru 00000080:00200000:1.0:1775764896.104339:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c2d71c89 23828: 1< 0 - - > 2< 97599488 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3014656 - | + - + - > 5< + - + - | 0 - | 3348 - -> 00000080:00200000:1.0:1775764896.104341:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000014762fad 00000080:00200000:1.0:1775764896.104342:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.104344:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.104352:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.104359:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.104366:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.104372:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.104383:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.104387:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.104389:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.104399:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.104402:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.104404:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.104408:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104416:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.104424:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.104426:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.104431:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.104434:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23828, 23828] 00000080:00200000:1.0:1775764896.104554:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.104558:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.104591:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97603584 i_blocks 90112 00000080:00200000:1.0:1775764896.104594:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.104612:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97603584, count: 4096 00000080:00200000:1.0:1775764896.104622:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97603584, bytes: 4096 00000080:00200000:1.0:1775764896.104625:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.104629:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23830, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.104631:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97603584, 97607680) 0 0 00000080:00200000:1.0:1775764896.104633:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.104639:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23829, 23829] 00020000:00200000:1.0:1775764896.104641:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97603584 chunk: [97603584, 97607680) 97607680, 4096 00020000:00200000:1.0:1775764896.104643:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.104654:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23829, 23829] 00000008:00010000:1.0:1775764896.104660:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.104662:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.104671:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104680:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97603584 97607679) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104688:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104695:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104703:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.104712:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97603584, 97607680) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.104718:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23829 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.104735:0:18302:0:(rw26.c:858:ll_write_end()) pos 97603584, len 4096, copied 4096 00000080:00200000:1.0:1775764896.104739:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000636d1656[3 00000000b99c1f5d 2 1 00000000155ed523] 00000080:00200000:1.0:1775764896.104742:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003f7eac22 17ffffc0001001 3:0 ffff90c20b3e8000 23829 lru 00000080:00200000:1.0:1775764896.104748:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000080c15cc5 23829: 1< 0 - - > 2< 97603584 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 3010560 - | + - + - > 5< + - + - | 0 - | 3349 - -> 00000080:00200000:1.0:1775764896.104750:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000636d1656 00000080:00200000:1.0:1775764896.104751:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.104753:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.104761:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.104769:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000395f3f9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.104776:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.104783:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.104795:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.104798:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.104801:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.104811:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.104814:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.104817:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.104820:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.104829:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.104838:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eac68 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.104841:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.104845:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.104848:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23829, 23829] 00000080:00200000:1.0:1775764896.104858:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.104860:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.121133:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97751040 i_blocks 90112 00000080:00200000:1.0:1775764896.121138:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.121159:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97751040, count: 4096 00000080:00200000:1.0:1775764896.121177:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97751040, bytes: 4096 00000080:00200000:1.0:1775764896.121180:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.121184:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23866, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.121185:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97751040, 97755136) 0 0 00000080:00200000:1.0:1775764896.121188:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.121194:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23865, 23865] 00020000:00200000:1.0:1775764896.121197:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97751040 chunk: [97751040, 97755136) 97755136, 4096 00020000:00200000:1.0:1775764896.121199:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.121217:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23865, 23865] 00000008:00010000:1.0:1775764896.121224:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.121226:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.121235:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121244:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97751040 97755135) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121253:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121260:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121267:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.121276:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97751040, 97755136) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.121281:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23865 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.121302:0:18302:0:(rw26.c:858:ll_write_end()) pos 97751040, len 4096, copied 4096 00000080:00200000:1.0:1775764896.121306:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b8b72f46[3 00000000b99c1f5d 2 1 000000007d1ced01] 00000080:00200000:1.0:1775764896.121309:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006e98f6fc 17ffffc0001001 3:0 ffff90c20b3e8ee0 23865 lru 00000080:00200000:1.0:1775764896.121316:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000772a0b67 23865: 1< 0 - - > 2< 97751040 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2863104 - | + - + - > 5< + - + - | 0 - | 3385 - -> 00000080:00200000:1.0:1775764896.121318:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b8b72f46 00000080:00200000:1.0:1775764896.121319:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.121321:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.121329:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.121337:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.121344:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.121357:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.121370:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.121373:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.121376:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.121388:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.121391:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.121394:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.121398:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121406:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.121415:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.121416:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.121420:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.121422:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23865, 23865] 00000080:00200000:1.0:1775764896.121437:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.121504:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.121529:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97755136 i_blocks 90112 00000080:00200000:1.0:1775764896.121532:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.121551:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97755136, count: 4096 00000080:00200000:1.0:1775764896.121561:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97755136, bytes: 4096 00000080:00200000:1.0:1775764896.121564:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.121568:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23867, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.121570:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97755136, 97759232) 0 0 00000080:00200000:1.0:1775764896.121572:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.121578:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23866, 23866] 00020000:00200000:1.0:1775764896.121581:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97755136 chunk: [97755136, 97759232) 97759232, 4096 00020000:00200000:1.0:1775764896.121583:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.121600:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23866, 23866] 00000008:00010000:1.0:1775764896.121605:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.121607:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.121615:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121625:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97755136 97759231) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121633:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121640:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121647:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.121655:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97755136, 97759232) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.121660:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23866 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.121680:0:18302:0:(rw26.c:858:ll_write_end()) pos 97755136, len 4096, copied 4096 00000080:00200000:1.0:1775764896.121684:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fa8060f1[3 00000000b99c1f5d 2 1 000000007d1ced01] 00000080:00200000:1.0:1775764896.121686:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007ad9a131 17ffffc0001001 3:0 ffff90c20b3e80e0 23866 lru 00000080:00200000:1.0:1775764896.121691:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003159a9c6 23866: 1< 0 - - > 2< 97755136 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2859008 - | + - + - > 5< + - + - | 0 - | 3386 - -> 00000080:00200000:1.0:1775764896.121694:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fa8060f1 00000080:00200000:1.0:1775764896.121695:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.121697:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.121705:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.121713:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.121720:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.121727:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.121738:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.121741:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.121743:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.121753:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.121756:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.121758:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.121762:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121770:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.121778:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.121781:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.121785:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.121788:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23866, 23866] 00000080:00200000:1.0:1775764896.121799:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.121802:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.121832:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97759232 i_blocks 90112 00000080:00200000:1.0:1775764896.121836:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.121852:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97759232, count: 4096 00000080:00200000:1.0:1775764896.121863:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97759232, bytes: 4096 00000080:00200000:1.0:1775764896.121866:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.121893:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23868, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.121894:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97759232, 97763328) 0 0 00000080:00200000:1.0:1775764896.121897:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.121902:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23867, 23867] 00020000:00200000:1.0:1775764896.121905:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97759232 chunk: [97759232, 97763328) 97763328, 4096 00020000:00200000:1.0:1775764896.121907:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.121920:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23867, 23867] 00000008:00010000:1.0:1775764896.121925:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.121928:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.121935:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121942:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97759232 97763327) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121951:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121959:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.121971:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.121979:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97759232, 97763328) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.121985:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23867 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.122002:0:18302:0:(rw26.c:858:ll_write_end()) pos 97759232, len 4096, copied 4096 00000080:00200000:1.0:1775764896.122006:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e389db4f[3 00000000b99c1f5d 2 1 000000007d1ced01] 00000080:00200000:1.0:1775764896.122009:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000806e7b72 17ffffc0001001 3:0 ffff90c20b3e81c0 23867 lru 00000080:00200000:1.0:1775764896.122014:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f2614065 23867: 1< 0 - - > 2< 97759232 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2854912 - | + - + - > 5< + - + - | 0 - | 3387 - -> 00000080:00200000:1.0:1775764896.122017:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e389db4f 00000080:00200000:1.0:1775764896.122018:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.122020:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.122028:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122036:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122043:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122050:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.122060:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.122063:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.122065:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.122075:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.122078:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.122080:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.122083:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122091:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.122098:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.122100:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.122104:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.122107:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23867, 23867] 00000080:00200000:1.0:1775764896.122130:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.122133:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.122162:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97763328 i_blocks 90112 00000080:00200000:1.0:1775764896.122166:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.122182:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97763328, count: 4096 00000080:00200000:1.0:1775764896.122190:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97763328, bytes: 4096 00000080:00200000:1.0:1775764896.122192:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.122194:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23869, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.122195:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97763328, 97767424) 0 0 00000080:00200000:1.0:1775764896.122197:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.122201:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23868, 23868] 00020000:00200000:1.0:1775764896.122203:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97763328 chunk: [97763328, 97767424) 97767424, 4096 00020000:00200000:1.0:1775764896.122205:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.122212:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23868, 23868] 00000008:00010000:1.0:1775764896.122215:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.122217:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.122223:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122229:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97763328 97767423) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122235:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122239:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122243:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.122247:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97763328, 97767424) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.122250:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23868 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.122264:0:18302:0:(rw26.c:858:ll_write_end()) pos 97763328, len 4096, copied 4096 00000080:00200000:1.0:1775764896.122267:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000013b79506[3 00000000b99c1f5d 2 1 000000007d1ced01] 00000080:00200000:1.0:1775764896.122269:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000099fdaa7a 17ffffc0001001 3:0 ffff90c20b3e87e0 23868 lru 00000080:00200000:1.0:1775764896.122273:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005438f2c8 23868: 1< 0 - - > 2< 97763328 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2850816 - | + - + - > 5< + - + - | 0 - | 3388 - -> 00000080:00200000:1.0:1775764896.122274:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000013b79506 00000080:00200000:1.0:1775764896.122275:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.122276:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.122281:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122286:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122291:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122296:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.122303:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.122304:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.122306:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.122312:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.122314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.122315:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.122317:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122322:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.122327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.122328:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.122331:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.122333:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23868, 23868] 00000080:00200000:1.0:1775764896.122340:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.122342:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.122360:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97767424 i_blocks 90112 00000080:00200000:1.0:1775764896.122363:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.122374:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97767424, count: 4096 00000080:00200000:1.0:1775764896.122380:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97767424, bytes: 4096 00000080:00200000:1.0:1775764896.122382:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.122385:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23870, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.122386:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97767424, 97771520) 0 0 00000080:00200000:1.0:1775764896.122388:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.122393:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23869, 23869] 00020000:00200000:1.0:1775764896.122395:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97767424 chunk: [97767424, 97771520) 97771520, 4096 00020000:00200000:1.0:1775764896.122398:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.122409:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23869, 23869] 00000008:00010000:1.0:1775764896.122413:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.122416:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.122423:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122432:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97767424 97771519) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122509:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122515:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122519:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.122524:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97767424, 97771520) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.122528:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23869 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.122544:0:18302:0:(rw26.c:858:ll_write_end()) pos 97767424, len 4096, copied 4096 00000080:00200000:1.0:1775764896.122548:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004f49e2aa[3 00000000b99c1f5d 2 1 000000007d1ced01] 00000080:00200000:1.0:1775764896.122551:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000073bfaf0a 17ffffc0001001 3:0 ffff90c20b3e89a0 23869 lru 00000080:00200000:1.0:1775764896.122556:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b82b6d95 23869: 1< 0 - - > 2< 97767424 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2846720 - | + - + - > 5< + - + - | 0 - | 3389 - -> 00000080:00200000:1.0:1775764896.122559:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004f49e2aa 00000080:00200000:1.0:1775764896.122560:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.122562:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.122570:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122577:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122585:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122593:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.122604:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.122608:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.122609:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.122617:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.122619:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.122620:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.122623:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122628:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.122634:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.122636:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.122638:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.122640:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23869, 23869] 00000080:00200000:1.0:1775764896.122648:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.122650:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.122669:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97771520 i_blocks 90112 00000080:00200000:1.0:1775764896.122672:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.122683:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97771520, count: 4096 00000080:00200000:1.0:1775764896.122689:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97771520, bytes: 4096 00000080:00200000:1.0:1775764896.122691:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.122693:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23871, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.122694:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97771520, 97775616) 0 0 00000080:00200000:1.0:1775764896.122696:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.122699:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23870, 23870] 00020000:00200000:1.0:1775764896.122701:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97771520 chunk: [97771520, 97775616) 97775616, 4096 00020000:00200000:1.0:1775764896.122703:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.122709:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23870, 23870] 00000008:00010000:1.0:1775764896.122713:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.122714:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.122720:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122726:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97771520 97775615) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122731:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122740:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122744:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.122748:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97771520, 97775616) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.122751:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23870 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.122763:0:18302:0:(rw26.c:858:ll_write_end()) pos 97771520, len 4096, copied 4096 00000080:00200000:1.0:1775764896.122766:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000212426c1[3 00000000b99c1f5d 2 1 000000007d1ced01] 00000080:00200000:1.0:1775764896.122768:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c5ae2bd4 17ffffc0001001 3:0 ffff90c20b3e82a0 23870 lru 00000080:00200000:1.0:1775764896.122771:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ac5eaf7f 23870: 1< 0 - - > 2< 97771520 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2842624 - | + - + - > 5< + - + - | 0 - | 3390 - -> 00000080:00200000:1.0:1775764896.122772:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000212426c1 00000080:00200000:1.0:1775764896.122773:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.122774:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.122779:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122784:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000d3e340d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122788:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.122792:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.122804:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.122807:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.122809:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.122817:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.122820:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.122822:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.122826:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.122834:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.122843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea7b8 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.122845:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.122849:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.122852:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23870, 23870] 00000080:00200000:1.0:1775764896.122862:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.122866:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00010000:00010000:0.0:1775764896.131184:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131193:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131197:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.131201:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97882112, 97886208) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.131204:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23897 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.131216:0:18302:0:(rw26.c:858:ll_write_end()) pos 97882112, len 4096, copied 4096 00000080:00200000:0.0:1775764896.131219:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e169508c[3 00000000b99c1f5d 2 1 00000000cd45786d] 00000080:00200000:0.0:1775764896.131221:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fc411602 17ffffc0001001 3:0 ffff90c2128110e0 23897 lru 00000080:00200000:0.0:1775764896.131224:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f724b12e 23897: 1< 0 - - > 2< 97882112 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2732032 - | + - + - > 5< + - + - | 0 - | 3417 - -> 00000080:00200000:0.0:1775764896.131225:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e169508c 00000080:00200000:0.0:1775764896.131226:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.131227:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.131232:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009f8801cb ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.131237:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009f8801cb ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.131242:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.131246:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.131256:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.131258:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.131259:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.131266:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1998 (ffff90c210c81e18/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.131268:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1998 (ffff90c210c81e18/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.131269:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.131272:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131277:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.131282:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1998 (ffff90c210c81e18/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.131283:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81e18/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.131286:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.131287:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23897, 23897] 00000080:00200000:0.0:1775764896.131295:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.131297:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.131316:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97886208 i_blocks 90112 00000080:00200000:0.0:1775764896.131318:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.131331:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97886208, count: 4096 00000080:00200000:0.0:1775764896.131341:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97886208, bytes: 4096 00000080:00200000:0.0:1775764896.131343:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.131346:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23899, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.131348:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97886208, 97890304) 0 0 00000080:00200000:0.0:1775764896.131350:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.131355:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23898, 23898] 00020000:00200000:0.0:1775764896.131358:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97886208 chunk: [97886208, 97890304) 97890304, 4096 00020000:00200000:0.0:1775764896.131360:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.131370:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23898, 23898] 00000008:00010000:0.0:1775764896.131375:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.131378:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.131385:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131394:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97886208 97890303) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131402:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131408:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131415:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.131422:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97886208, 97890304) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.131428:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23898 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.131533:0:18302:0:(rw26.c:858:ll_write_end()) pos 97886208, len 4096, copied 4096 00000080:00200000:0.0:1775764896.131538:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003e4225fc[3 00000000b99c1f5d 2 1 00000000cd45786d] 00000080:00200000:0.0:1775764896.131541:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002dbde55d 17ffffc0001001 3:0 ffff90c212812a80 23898 lru 00000080:00200000:0.0:1775764896.131546:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000046e06ef3 23898: 1< 0 - - > 2< 97886208 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2727936 - | + - + - > 5< + - + - | 0 - | 3418 - -> 00000080:00200000:0.0:1775764896.131548:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003e4225fc 00000080:00200000:0.0:1775764896.131549:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.131551:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.131559:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009f8801cb ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.131567:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009f8801cb ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.131573:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.131580:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.131591:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.131595:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.131597:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.131607:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1998 (ffff90c210c81e18/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.131609:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1998 (ffff90c210c81e18/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.131611:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.131614:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131619:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.131625:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1998 (ffff90c210c81e18/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.131626:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81e18/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.131629:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.131631:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23898, 23898] 00000080:00200000:0.0:1775764896.131639:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.131641:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.131660:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97890304 i_blocks 90112 00000080:00200000:0.0:1775764896.131663:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.131673:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97890304, count: 4096 00000080:00200000:0.0:1775764896.131680:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97890304, bytes: 4096 00000080:00200000:0.0:1775764896.131682:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.131684:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23900, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.131685:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97890304, 97894400) 0 0 00000080:00200000:0.0:1775764896.131687:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.131690:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23899, 23899] 00020000:00200000:0.0:1775764896.131692:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97890304 chunk: [97890304, 97894400) 97894400, 4096 00020000:00200000:0.0:1775764896.131693:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.131700:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23899, 23899] 00000008:00010000:0.0:1775764896.131704:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.131705:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.131710:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131716:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97890304 97894399) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131722:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131727:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131730:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.131736:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97890304, 97894400) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.131739:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23899 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.131761:0:18302:0:(rw26.c:858:ll_write_end()) pos 97890304, len 4096, copied 4096 00000080:00200000:0.0:1775764896.131765:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004ea6df23[3 00000000b99c1f5d 2 1 00000000cd45786d] 00000080:00200000:0.0:1775764896.131768:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002175bb3c 17ffffc0001021 2:0 ffff90c212812700 23899 lru 00000080:00200000:0.0:1775764896.131773:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007d2494cd 23899: 1< 0 - - > 2< 97890304 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2723840 - | + - + - > 5< + - + - | 0 - | 3419 - -> 00000080:00200000:0.0:1775764896.131775:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004ea6df23 00000080:00200000:0.0:1775764896.131776:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.131778:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.131785:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009f8801cb ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.131792:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009f8801cb ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.131798:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.131804:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.131814:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.131816:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.131818:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.131827:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1998 (ffff90c210c81e18/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.131829:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1998 (ffff90c210c81e18/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.131832:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.131836:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.131842:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.131850:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1998 (ffff90c210c81e18/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.131852:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81e18/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.131856:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.131858:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23899, 23899] 00000080:00200000:0.0:1775764896.131964:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.131968:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.132130:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97894400 i_blocks 90112 00000080:00200000:1.0:1775764896.132135:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.132156:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97894400, count: 4096 00000080:00200000:1.0:1775764896.132174:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97894400, bytes: 4096 00000080:00200000:1.0:1775764896.132177:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.132181:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23901, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.132183:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97894400, 97898496) 0 0 00000080:00200000:1.0:1775764896.132185:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.132191:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23900, 23900] 00020000:00200000:1.0:1775764896.132194:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97894400 chunk: [97894400, 97898496) 97898496, 4096 00020000:00200000:1.0:1775764896.132197:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.132214:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23900, 23900] 00000008:00010000:1.0:1775764896.132221:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.132224:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.132232:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132241:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97894400 97898495) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132249:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132256:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132262:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.132271:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97894400, 97898496) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.132276:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23900 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.132295:0:18302:0:(rw26.c:858:ll_write_end()) pos 97894400, len 4096, copied 4096 00000080:00200000:1.0:1775764896.132300:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000078773216[3 00000000b99c1f5d 2 1 00000000cdcfb5ff] 00000080:00200000:1.0:1775764896.132302:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000850669c 17ffffc0001001 3:0 ffff90c20b3e8d20 23900 lru 00000080:00200000:1.0:1775764896.132308:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008bfbe268 23900: 1< 0 - - > 2< 97894400 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2719744 - | + - + - > 5< + - + - | 0 - | 3420 - -> 00000080:00200000:1.0:1775764896.132310:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000078773216 00000080:00200000:1.0:1775764896.132311:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.132313:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.132321:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ffdb64d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.132330:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ffdb64d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.132337:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.132344:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.132355:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.132358:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.132360:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.132374:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.132376:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.132379:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.132382:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132390:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.132397:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.132399:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.132403:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.132406:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23900, 23900] 00000080:00200000:1.0:1775764896.132424:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.132427:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.132554:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97898496 i_blocks 90112 00000080:00200000:1.0:1775764896.132558:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.132576:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97898496, count: 4096 00000080:00200000:1.0:1775764896.132586:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97898496, bytes: 4096 00000080:00200000:1.0:1775764896.132588:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.132592:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23902, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.132594:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97898496, 97902592) 0 0 00000080:00200000:1.0:1775764896.132596:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.132602:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23901, 23901] 00020000:00200000:1.0:1775764896.132605:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97898496 chunk: [97898496, 97902592) 97902592, 4096 00020000:00200000:1.0:1775764896.132607:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.132618:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23901, 23901] 00000008:00010000:1.0:1775764896.132623:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.132626:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.132633:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132642:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97898496 97902591) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132650:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132658:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132665:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.132674:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97898496, 97902592) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.132679:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23901 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.132729:0:18302:0:(rw26.c:858:ll_write_end()) pos 97898496, len 4096, copied 4096 00000080:00200000:1.0:1775764896.132734:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000053dc11e8[3 00000000b99c1f5d 2 1 00000000cdcfb5ff] 00000080:00200000:1.0:1775764896.132737:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007db1fd8f 17ffffc0001021 2:0 ffff90c20b3e8380 23901 lru 00000080:00200000:1.0:1775764896.132743:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004d3269d3 23901: 1< 0 - - > 2< 97898496 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2715648 - | + - + - > 5< + - + - | 0 - | 3421 - -> 00000080:00200000:1.0:1775764896.132745:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000053dc11e8 00000080:00200000:1.0:1775764896.132746:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.132748:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.132756:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ffdb64d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.132764:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ffdb64d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.132771:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.132778:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.132790:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.132792:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.132794:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.132806:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.132808:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.132811:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.132816:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132824:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.132832:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.132835:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.132839:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.132842:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23901, 23901] 00000080:00200000:1.0:1775764896.132853:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.132857:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.132908:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97902592 i_blocks 90112 00000080:00200000:1.0:1775764896.132913:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.132929:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97902592, count: 4096 00000080:00200000:1.0:1775764896.132940:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97902592, bytes: 4096 00000080:00200000:1.0:1775764896.132943:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.132946:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23903, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.132948:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97902592, 97906688) 0 0 00000080:00200000:1.0:1775764896.132950:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.132956:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23902, 23902] 00020000:00200000:1.0:1775764896.132959:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97902592 chunk: [97902592, 97906688) 97906688, 4096 00020000:00200000:1.0:1775764896.132961:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.132974:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23902, 23902] 00000008:00010000:1.0:1775764896.132979:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.132982:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.132990:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.132999:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97902592 97906687) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.133008:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.133016:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.133023:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.133031:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97902592, 97906688) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.133036:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23902 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.133061:0:18302:0:(rw26.c:858:ll_write_end()) pos 97902592, len 4096, copied 4096 00000080:00200000:1.0:1775764896.133066:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b5d6e436[3 00000000b99c1f5d 2 1 00000000cdcfb5ff] 00000080:00200000:1.0:1775764896.133068:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008f6410b6 17ffffc0001001 3:0 ffff90c2128132a0 23902 lru 00000080:00200000:1.0:1775764896.133075:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000059ff0dd6 23902: 1< 0 - - > 2< 97902592 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2711552 - | + - + - > 5< + - + - | 0 - | 3422 - -> 00000080:00200000:1.0:1775764896.133077:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b5d6e436 00000080:00200000:1.0:1775764896.133078:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.133080:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.133088:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ffdb64d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.133097:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ffdb64d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.133104:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.133130:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.133142:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.133145:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.133147:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.133158:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.133161:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.133163:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.133167:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.133176:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.133183:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.133186:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.133190:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.133193:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23902, 23902] 00000080:00200000:1.0:1775764896.133204:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.133207:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.133240:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 97906688 i_blocks 90112 00000080:00200000:1.0:1775764896.133243:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.133259:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 97906688, count: 4096 00000080:00200000:1.0:1775764896.133270:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 97906688, bytes: 4096 00000080:00200000:1.0:1775764896.133273:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.133278:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23904, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.133279:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [97906688, 97910784) 0 0 00000080:00200000:1.0:1775764896.133282:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.133288:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23903, 23903] 00020000:00200000:1.0:1775764896.133291:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 97906688 chunk: [97906688, 97910784) 97910784, 4096 00020000:00200000:1.0:1775764896.133293:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.133306:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23903, 23903] 00000008:00010000:1.0:1775764896.133312:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.133314:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.133323:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.133332:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (97906688 97910783) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.133341:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.133348:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.133355:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.133363:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [97906688, 97910784) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.133368:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23903 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.133386:0:18302:0:(rw26.c:858:ll_write_end()) pos 97906688, len 4096, copied 4096 00000080:00200000:1.0:1775764896.133391:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000879aa4df[3 00000000b99c1f5d 2 1 00000000cdcfb5ff] 00000080:00200000:1.0:1775764896.133394:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a009d66a 17ffffc0001001 3:0 ffff90c212813d20 23903 lru 00000080:00200000:1.0:1775764896.133399:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b2e1bb79 23903: 1< 0 - - > 2< 97906688 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2707456 - | + - + - > 5< + - + - | 0 - | 3423 - -> 00000080:00200000:1.0:1775764896.133402:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000879aa4df 00000080:00200000:1.0:1775764896.133403:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.133405:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.133414:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ffdb64d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.133422:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ffdb64d8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.133430:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.133437:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.133748:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.133751:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.133753:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.133764:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.133767:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.133770:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.133774:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.133781:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.133789:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea8a8 (ffff90c2195eebd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.133792:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eebd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.133796:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.133800:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23903, 23903] 00000080:00200000:1.0:1775764896.133811:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.133814:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.148035:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98070528 i_blocks 90112 00000080:00200000:3.0:1775764896.148041:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.148070:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98070528, count: 4096 00000080:00200000:3.0:1775764896.148096:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98070528, bytes: 4096 00000080:00200000:3.0:1775764896.148101:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.148105:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23944, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.148106:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98070528, 98074624) 0 0 00000080:00200000:3.0:1775764896.148110:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.148132:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23943, 23943] 00020000:00200000:3.0:1775764896.148139:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98070528 chunk: [98070528, 98074624) 98074624, 4096 00020000:00200000:3.0:1775764896.148142:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.148164:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23943, 23943] 00000008:00010000:3.0:1775764896.148173:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.148177:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.148190:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148201:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98070528 98074623) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148211:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148224:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148231:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.148240:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98070528, 98074624) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.148248:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23943 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.148284:0:18302:0:(rw26.c:858:ll_write_end()) pos 98070528, len 4096, copied 4096 00000080:00200000:3.0:1775764896.148289:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000013a00e6d[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.148291:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007685fb0e 17ffffc0001001 3:0 ffff90c21280c540 23943 lru 00000080:00200000:3.0:1775764896.148298:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000dc6bd230 23943: 1< 0 - - > 2< 98070528 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2543616 - | + - + - > 5< + - + - | 0 - | 3463 - -> 00000080:00200000:3.0:1775764896.148300:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000013a00e6d 00000080:00200000:3.0:1775764896.148301:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.148303:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.148312:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.148320:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.148327:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.148335:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.148353:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.148356:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.148359:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.148367:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.148370:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.148372:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.148377:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148385:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.148396:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.148398:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.148403:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.148406:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23943, 23943] 00000080:00200000:3.0:1775764896.148425:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.148428:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.148433:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98074624 i_blocks 90112 00000080:00200000:3.0:1775764896.148435:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.148468:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98074624, count: 4096 00000080:00200000:3.0:1775764896.148481:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98074624, bytes: 4096 00000080:00200000:3.0:1775764896.148484:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.148487:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23945, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.148489:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98074624, 98078720) 0 0 00000080:00200000:3.0:1775764896.148491:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.148497:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23944, 23944] 00020000:00200000:3.0:1775764896.148501:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98074624 chunk: [98074624, 98078720) 98078720, 4096 00020000:00200000:3.0:1775764896.148503:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.148515:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23944, 23944] 00000008:00010000:3.0:1775764896.148520:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.148523:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.148531:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148540:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98074624 98078719) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148548:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148556:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148562:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.148571:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98074624, 98078720) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.148577:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23944 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.148597:0:18302:0:(rw26.c:858:ll_write_end()) pos 98074624, len 4096, copied 4096 00000080:00200000:3.0:1775764896.148602:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000eb51620f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.148604:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000081cb2c94 17ffffc0001001 3:0 ffff90c21280c460 23944 lru 00000080:00200000:3.0:1775764896.148610:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009247f4f2 23944: 1< 0 - - > 2< 98074624 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2539520 - | + - + - > 5< + - + - | 0 - | 3464 - -> 00000080:00200000:3.0:1775764896.148612:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000eb51620f 00000080:00200000:3.0:1775764896.148614:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.148616:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.148625:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.148632:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.148640:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.148647:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.148658:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.148661:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.148663:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.148669:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.148671:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.148674:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.148678:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148686:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.148694:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.148696:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.148701:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.148704:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23944, 23944] 00000080:00200000:3.0:1775764896.148716:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.148719:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.148724:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98078720 i_blocks 90112 00000080:00200000:3.0:1775764896.148726:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.148742:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98078720, count: 4096 00000080:00200000:3.0:1775764896.148753:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98078720, bytes: 4096 00000080:00200000:3.0:1775764896.148756:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.148760:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23946, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.148762:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98078720, 98082816) 0 0 00000080:00200000:3.0:1775764896.148764:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.148770:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23945, 23945] 00020000:00200000:3.0:1775764896.148774:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98078720 chunk: [98078720, 98082816) 98082816, 4096 00020000:00200000:3.0:1775764896.148776:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.148788:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23945, 23945] 00000008:00010000:3.0:1775764896.148794:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.148796:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.148805:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148814:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98078720 98082815) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148822:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148829:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.148836:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.148845:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98078720, 98082816) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.148850:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23945 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.148913:0:18302:0:(rw26.c:858:ll_write_end()) pos 98078720, len 4096, copied 4096 00000080:00200000:3.0:1775764896.148918:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005dbe9659[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.148921:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fdba2bca 17ffffc0001001 3:0 ffff90c212819460 23945 lru 00000080:00200000:3.0:1775764896.148927:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000044ba49a9 23945: 1< 0 - - > 2< 98078720 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2535424 - | + - + - > 5< + - + - | 0 - | 3465 - -> 00000080:00200000:3.0:1775764896.148930:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005dbe9659 00000080:00200000:3.0:1775764896.148931:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.148961:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.148971:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.148979:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.148986:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.148994:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.149006:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.149009:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.149012:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.149017:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.149020:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.149023:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.149028:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149035:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.149044:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.149046:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.149050:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.149053:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23945, 23945] 00000080:00200000:3.0:1775764896.149065:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.149068:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.149073:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98082816 i_blocks 90112 00000080:00200000:3.0:1775764896.149075:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.149093:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98082816, count: 4096 00000080:00200000:3.0:1775764896.149105:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98082816, bytes: 4096 00000080:00200000:3.0:1775764896.149108:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.149112:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23947, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.149127:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98082816, 98086912) 0 0 00000080:00200000:3.0:1775764896.149130:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.149136:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23946, 23946] 00020000:00200000:3.0:1775764896.149139:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98082816 chunk: [98082816, 98086912) 98086912, 4096 00020000:00200000:3.0:1775764896.149141:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.149154:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23946, 23946] 00000008:00010000:3.0:1775764896.149160:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.149162:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.149169:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149178:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98082816 98086911) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149186:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149193:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149200:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.149206:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98082816, 98086912) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.149210:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23946 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.149227:0:18302:0:(rw26.c:858:ll_write_end()) pos 98082816, len 4096, copied 4096 00000080:00200000:3.0:1775764896.149231:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d6e88dd9[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.149233:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000dd135771 17ffffc0001001 3:0 ffff90c212819a80 23946 lru 00000080:00200000:3.0:1775764896.149239:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001c7ec65f 23946: 1< 0 - - > 2< 98082816 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2531328 - | + - + - > 5< + - + - | 0 - | 3466 - -> 00000080:00200000:3.0:1775764896.149241:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d6e88dd9 00000080:00200000:3.0:1775764896.149242:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.149244:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.149252:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.149261:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.149269:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.149282:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.149293:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.149295:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.149298:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.149303:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.149305:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.149308:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.149311:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149319:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.149327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.149329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.149333:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.149336:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23946, 23946] 00000080:00200000:3.0:1775764896.149346:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.149350:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.149354:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98086912 i_blocks 90112 00000080:00200000:3.0:1775764896.149357:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.149370:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98086912, count: 4096 00000080:00200000:3.0:1775764896.149381:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98086912, bytes: 4096 00000080:00200000:3.0:1775764896.149383:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.149387:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23948, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.149389:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98086912, 98091008) 0 0 00000080:00200000:3.0:1775764896.149391:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.149396:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23947, 23947] 00020000:00200000:3.0:1775764896.149399:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98086912 chunk: [98086912, 98091008) 98091008, 4096 00020000:00200000:3.0:1775764896.149401:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.149412:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23947, 23947] 00000008:00010000:3.0:1775764896.149417:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.149419:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.149426:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149435:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98086912 98091007) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149444:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149451:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149459:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.149467:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98086912, 98091008) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.149473:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23947 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.149491:0:18302:0:(rw26.c:858:ll_write_end()) pos 98086912, len 4096, copied 4096 00000080:00200000:3.0:1775764896.149496:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000097cb47a3[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.149499:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a2396d9a 17ffffc0001001 3:0 ffff90c212819700 23947 lru 00000080:00200000:3.0:1775764896.149505:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c2f8d3fe 23947: 1< 0 - - > 2< 98086912 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2527232 - | + - + - > 5< + - + - | 0 - | 3467 - -> 00000080:00200000:3.0:1775764896.149508:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000097cb47a3 00000080:00200000:3.0:1775764896.149509:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.149512:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.149520:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.149528:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.149536:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.149544:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.149555:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.149558:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.149561:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.149566:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.149569:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.149572:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.149576:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149584:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.149592:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.149596:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.149600:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.149603:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23947, 23947] 00000080:00200000:3.0:1775764896.149615:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.149618:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.149623:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98091008 i_blocks 90112 00000080:00200000:3.0:1775764896.149626:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.149641:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98091008, count: 4096 00000080:00200000:3.0:1775764896.149653:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98091008, bytes: 4096 00000080:00200000:3.0:1775764896.149656:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.149660:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23949, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.149662:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98091008, 98095104) 0 0 00000080:00200000:3.0:1775764896.149664:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.149670:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23948, 23948] 00020000:00200000:3.0:1775764896.149673:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98091008 chunk: [98091008, 98095104) 98095104, 4096 00020000:00200000:3.0:1775764896.149675:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.149687:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23948, 23948] 00000008:00010000:3.0:1775764896.149691:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.149694:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.149702:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149710:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98091008 98095103) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149718:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149727:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149740:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.149748:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98091008, 98095104) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.149753:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23948 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.149770:0:18302:0:(rw26.c:858:ll_write_end()) pos 98091008, len 4096, copied 4096 00000080:00200000:3.0:1775764896.149775:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e692eb7a[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.149778:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d1c7284d 17ffffc0001001 3:0 ffff90c212819c40 23948 lru 00000080:00200000:3.0:1775764896.149784:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c46903e5 23948: 1< 0 - - > 2< 98091008 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2523136 - | + - + - > 5< + - + - | 0 - | 3468 - -> 00000080:00200000:3.0:1775764896.149786:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e692eb7a 00000080:00200000:3.0:1775764896.149787:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.149789:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.149797:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.149805:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.149812:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.149820:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.149832:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.149835:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.149837:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.149843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.149846:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.149849:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.149852:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.149861:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.149896:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.149899:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.149904:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.149906:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23948, 23948] 00000080:00200000:3.0:1775764896.149918:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.149922:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.149927:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98095104 i_blocks 90112 00000080:00200000:3.0:1775764896.149929:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.149943:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98095104, count: 4096 00000080:00200000:3.0:1775764896.149953:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98095104, bytes: 4096 00000080:00200000:3.0:1775764896.149956:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.149959:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23950, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.149961:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98095104, 98099200) 0 0 00000080:00200000:3.0:1775764896.149963:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.149969:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23949, 23949] 00020000:00200000:3.0:1775764896.149972:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98095104 chunk: [98095104, 98099200) 98099200, 4096 00020000:00200000:3.0:1775764896.149974:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.149984:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23949, 23949] 00000008:00010000:3.0:1775764896.149989:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.149992:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.149999:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150008:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98095104 98099199) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150016:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150023:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150030:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.150038:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98095104, 98099200) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.150042:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23949 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.150058:0:18302:0:(rw26.c:858:ll_write_end()) pos 98095104, len 4096, copied 4096 00000080:00200000:3.0:1775764896.150062:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a5850f5f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.150065:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000358611e1 17ffffc0001001 3:0 ffff90c2128198c0 23949 lru 00000080:00200000:3.0:1775764896.150070:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004d889213 23949: 1< 0 - - > 2< 98095104 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2519040 - | + - + - > 5< + - + - | 0 - | 3469 - -> 00000080:00200000:3.0:1775764896.150072:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a5850f5f 00000080:00200000:3.0:1775764896.150073:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.150075:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.150082:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150091:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150098:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150106:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.150131:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.150134:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.150137:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.150142:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.150145:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.150147:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.150151:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150159:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.150168:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.150170:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.150174:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.150177:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23949, 23949] 00000080:00200000:3.0:1775764896.150187:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.150190:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.150194:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98099200 i_blocks 90112 00000080:00200000:3.0:1775764896.150197:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.150212:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98099200, count: 4096 00000080:00200000:3.0:1775764896.150222:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98099200, bytes: 4096 00000080:00200000:3.0:1775764896.150224:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.150228:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23951, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.150229:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98099200, 98103296) 0 0 00000080:00200000:3.0:1775764896.150232:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.150237:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23950, 23950] 00020000:00200000:3.0:1775764896.150240:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98099200 chunk: [98099200, 98103296) 98103296, 4096 00020000:00200000:3.0:1775764896.150242:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.150254:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23950, 23950] 00000008:00010000:3.0:1775764896.150259:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.150261:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.150269:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150277:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98099200 98103295) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150285:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150292:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150299:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.150306:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98099200, 98103296) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.150311:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23950 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.150330:0:18302:0:(rw26.c:858:ll_write_end()) pos 98099200, len 4096, copied 4096 00000080:00200000:3.0:1775764896.150335:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003541c887[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.150337:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005ec946de 17ffffc0001001 3:0 ffff90c212819b60 23950 lru 00000080:00200000:3.0:1775764896.150343:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e7b4e474 23950: 1< 0 - - > 2< 98099200 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2514944 - | + - + - > 5< + - + - | 0 - | 3470 - -> 00000080:00200000:3.0:1775764896.150346:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003541c887 00000080:00200000:3.0:1775764896.150346:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.150349:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.150357:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150365:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150372:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150379:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.150390:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.150393:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.150395:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.150401:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.150404:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.150406:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.150410:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150418:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.150427:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.150429:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.150433:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.150436:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23950, 23950] 00000080:00200000:3.0:1775764896.150447:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.150450:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.150455:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98103296 i_blocks 90112 00000080:00200000:3.0:1775764896.150457:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.150472:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98103296, count: 4096 00000080:00200000:3.0:1775764896.150483:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98103296, bytes: 4096 00000080:00200000:3.0:1775764896.150485:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.150489:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23952, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.150490:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98103296, 98107392) 0 0 00000080:00200000:3.0:1775764896.150492:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.150498:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23951, 23951] 00020000:00200000:3.0:1775764896.150501:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98103296 chunk: [98103296, 98107392) 98107392, 4096 00020000:00200000:3.0:1775764896.150503:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.150514:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23951, 23951] 00000008:00010000:3.0:1775764896.150519:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.150521:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.150528:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150537:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98103296 98107391) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150544:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150557:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150564:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.150572:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98103296, 98107392) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.150576:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23951 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.150600:0:18302:0:(rw26.c:858:ll_write_end()) pos 98103296, len 4096, copied 4096 00000080:00200000:3.0:1775764896.150604:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ea8d7529[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.150607:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009e891d6b 17ffffc0001021 2:0 ffff90c212819000 23951 lru 00000080:00200000:3.0:1775764896.150613:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003890f5e7 23951: 1< 0 - - > 2< 98103296 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2510848 - | + - + - > 5< + - + - | 0 - | 3471 - -> 00000080:00200000:3.0:1775764896.150615:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ea8d7529 00000080:00200000:3.0:1775764896.150616:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.150618:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.150626:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150633:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150640:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150647:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.150663:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.150665:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.150668:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.150673:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.150676:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.150678:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.150682:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150690:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.150698:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.150700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.150704:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.150706:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23951, 23951] 00000080:00200000:3.0:1775764896.150717:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.150720:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.150725:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98107392 i_blocks 90112 00000080:00200000:3.0:1775764896.150728:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.150743:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98107392, count: 4096 00000080:00200000:3.0:1775764896.150753:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98107392, bytes: 4096 00000080:00200000:3.0:1775764896.150756:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.150760:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23953, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.150761:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98107392, 98111488) 0 0 00000080:00200000:3.0:1775764896.150763:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.150769:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23952, 23952] 00020000:00200000:3.0:1775764896.150772:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98107392 chunk: [98107392, 98111488) 98111488, 4096 00020000:00200000:3.0:1775764896.150775:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.150786:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23952, 23952] 00000008:00010000:3.0:1775764896.150791:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.150794:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.150801:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150810:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98107392 98111487) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150818:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150826:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150833:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.150840:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98107392, 98111488) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.150845:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23952 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.150861:0:18302:0:(rw26.c:858:ll_write_end()) pos 98107392, len 4096, copied 4096 00000080:00200000:3.0:1775764896.150865:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000050d95f3a[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.150894:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001748442d 17ffffc0001001 3:0 ffff90c212819ee0 23952 lru 00000080:00200000:3.0:1775764896.150900:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b9f8e96d 23952: 1< 0 - - > 2< 98107392 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2506752 - | + - + - > 5< + - + - | 0 - | 3472 - -> 00000080:00200000:3.0:1775764896.150902:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000050d95f3a 00000080:00200000:3.0:1775764896.150903:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.150906:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.150915:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150923:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150931:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.150938:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.150949:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.150952:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.150954:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.150960:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.150962:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.150964:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.150968:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.150976:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.150984:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.150986:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.150991:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.150993:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23952, 23952] 00000080:00200000:3.0:1775764896.151005:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.151009:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.151014:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98111488 i_blocks 90112 00000080:00200000:3.0:1775764896.151016:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.151032:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98111488, count: 4096 00000080:00200000:3.0:1775764896.151044:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98111488, bytes: 4096 00000080:00200000:3.0:1775764896.151047:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.151051:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23954, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.151052:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98111488, 98115584) 0 0 00000080:00200000:3.0:1775764896.151055:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.151060:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23953, 23953] 00020000:00200000:3.0:1775764896.151063:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98111488 chunk: [98111488, 98115584) 98115584, 4096 00020000:00200000:3.0:1775764896.151065:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.151077:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23953, 23953] 00000008:00010000:3.0:1775764896.151082:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.151084:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.151092:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.151101:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98111488 98115583) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.151110:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.151618:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.151627:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.151638:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98111488, 98115584) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.151646:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23953 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.151670:0:18302:0:(rw26.c:858:ll_write_end()) pos 98111488, len 4096, copied 4096 00000080:00200000:3.0:1775764896.151675:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cac34efc[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.151679:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000bc296140 17ffffc0001001 3:0 ffff90c2128190e0 23953 lru 00000080:00200000:3.0:1775764896.151685:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009545d470 23953: 1< 0 - - > 2< 98111488 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2502656 - | + - + - > 5< + - + - | 0 - | 3473 - -> 00000080:00200000:3.0:1775764896.151687:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cac34efc 00000080:00200000:3.0:1775764896.151688:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.151690:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.151700:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.151707:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.151713:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.151720:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.151735:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.151738:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.151740:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.151762:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.151765:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.151767:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.151772:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.151780:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.151791:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.151793:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.151798:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.151801:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23953, 23953] 00000080:00200000:3.0:1775764896.151815:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.151818:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.151858:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98115584 i_blocks 90112 00000080:00200000:3.0:1775764896.151862:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.151899:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98115584, count: 4096 00000080:00200000:3.0:1775764896.151910:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98115584, bytes: 4096 00000080:00200000:3.0:1775764896.151914:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.151917:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23955, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.151919:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98115584, 98119680) 0 0 00000080:00200000:3.0:1775764896.151921:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.151929:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23954, 23954] 00020000:00200000:3.0:1775764896.151932:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98115584 chunk: [98115584, 98119680) 98119680, 4096 00020000:00200000:3.0:1775764896.151935:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.151946:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23954, 23954] 00000008:00010000:3.0:1775764896.151953:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.151956:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.151965:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.151975:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98115584 98119679) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.151984:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.151992:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.151999:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.152006:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98115584, 98119680) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.152011:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23954 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.152027:0:18302:0:(rw26.c:858:ll_write_end()) pos 98115584, len 4096, copied 4096 00000080:00200000:3.0:1775764896.152031:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001f5f1cca[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.152034:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003a2015b4 17ffffc0001001 3:0 ffff90c2128191c0 23954 lru 00000080:00200000:3.0:1775764896.152039:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005b710fbd 23954: 1< 0 - - > 2< 98115584 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2498560 - | + - + - > 5< + - + - | 0 - | 3474 - -> 00000080:00200000:3.0:1775764896.152041:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001f5f1cca 00000080:00200000:3.0:1775764896.152042:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.152044:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.152051:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152058:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152064:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152076:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.152085:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.152088:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.152090:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.152099:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.152102:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.152104:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.152108:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152127:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.152135:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.152137:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.152141:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.152144:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23954, 23954] 00000080:00200000:3.0:1775764896.152153:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.152156:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.152182:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98119680 i_blocks 90112 00000080:00200000:3.0:1775764896.152185:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.152199:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98119680, count: 4096 00000080:00200000:3.0:1775764896.152207:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98119680, bytes: 4096 00000080:00200000:3.0:1775764896.152209:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.152213:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23956, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.152215:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98119680, 98123776) 0 0 00000080:00200000:3.0:1775764896.152217:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.152222:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23955, 23955] 00020000:00200000:3.0:1775764896.152224:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98119680 chunk: [98119680, 98123776) 98123776, 4096 00020000:00200000:3.0:1775764896.152226:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.152236:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23955, 23955] 00000008:00010000:3.0:1775764896.152241:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.152243:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.152251:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152260:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98119680 98123775) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152267:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152274:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152281:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.152289:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98119680, 98123776) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.152294:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23955 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.152310:0:18302:0:(rw26.c:858:ll_write_end()) pos 98119680, len 4096, copied 4096 00000080:00200000:3.0:1775764896.152314:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009499b053[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.152317:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000658dc184 17ffffc0001001 3:0 ffff90c2128197e0 23955 lru 00000080:00200000:3.0:1775764896.152322:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000af8684e8 23955: 1< 0 - - > 2< 98119680 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2494464 - | + - + - > 5< + - + - | 0 - | 3475 - -> 00000080:00200000:3.0:1775764896.152325:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009499b053 00000080:00200000:3.0:1775764896.152326:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.152328:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.152336:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152344:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152352:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152360:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.152371:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.152373:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.152376:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.152385:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.152388:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.152391:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.152395:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152403:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.152412:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.152415:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.152419:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.152422:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23955, 23955] 00000080:00200000:3.0:1775764896.152433:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.152437:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.152528:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98123776 i_blocks 90112 00000080:00200000:3.0:1775764896.152531:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.152542:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98123776, count: 4096 00000080:00200000:3.0:1775764896.152548:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98123776, bytes: 4096 00000080:00200000:3.0:1775764896.152550:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.152552:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23957, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.152554:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98123776, 98127872) 0 0 00000080:00200000:3.0:1775764896.152555:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.152559:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23956, 23956] 00020000:00200000:3.0:1775764896.152561:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98123776 chunk: [98123776, 98127872) 98127872, 4096 00020000:00200000:3.0:1775764896.152563:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.152570:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23956, 23956] 00000008:00010000:3.0:1775764896.152573:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.152575:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.152581:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152587:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98123776 98127871) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152592:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152597:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152605:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.152610:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98123776, 98127872) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.152613:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23956 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.152626:0:18302:0:(rw26.c:858:ll_write_end()) pos 98123776, len 4096, copied 4096 00000080:00200000:3.0:1775764896.152628:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d68c9127[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.152630:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000dd203a8 17ffffc0001001 3:0 ffff90c2128199a0 23956 lru 00000080:00200000:3.0:1775764896.152634:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cf3a1a53 23956: 1< 0 - - > 2< 98123776 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2490368 - | + - + - > 5< + - + - | 0 - | 3476 - -> 00000080:00200000:3.0:1775764896.152635:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d68c9127 00000080:00200000:3.0:1775764896.152636:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.152637:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.152643:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152647:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152652:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152656:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.152663:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.152664:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.152666:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.152673:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.152675:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.152677:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.152679:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152684:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.152689:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.152690:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.152693:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.152694:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23956, 23956] 00000080:00200000:3.0:1775764896.152702:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.152704:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.152722:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98127872 i_blocks 90112 00000080:00200000:3.0:1775764896.152725:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.152735:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98127872, count: 4096 00000080:00200000:3.0:1775764896.152741:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98127872, bytes: 4096 00000080:00200000:3.0:1775764896.152743:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.152745:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23958, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.152746:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98127872, 98131968) 0 0 00000080:00200000:3.0:1775764896.152748:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.152751:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23957, 23957] 00020000:00200000:3.0:1775764896.152753:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98127872 chunk: [98127872, 98131968) 98131968, 4096 00020000:00200000:3.0:1775764896.152755:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.152761:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23957, 23957] 00000008:00010000:3.0:1775764896.152764:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.152766:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.152771:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152777:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98127872 98131967) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152783:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152787:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152791:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.152796:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98127872, 98131968) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.152799:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23957 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.152811:0:18302:0:(rw26.c:858:ll_write_end()) pos 98127872, len 4096, copied 4096 00000080:00200000:3.0:1775764896.152814:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005d1e0e43[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.152816:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a727a435 17ffffc0001001 3:0 ffff90c2128192a0 23957 lru 00000080:00200000:3.0:1775764896.152820:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000086cced46 23957: 1< 0 - - > 2< 98127872 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2486272 - | + - + - > 5< + - + - | 0 - | 3477 - -> 00000080:00200000:3.0:1775764896.152821:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005d1e0e43 00000080:00200000:3.0:1775764896.152822:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.152823:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.152828:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152833:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152838:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.152843:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.152849:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.152851:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.152852:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.152858:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.152860:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.152861:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.152863:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.152956:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.152966:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.152969:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.152973:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.152976:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23957, 23957] 00000080:00200000:3.0:1775764896.152987:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.152990:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.153018:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98131968 i_blocks 90112 00000080:00200000:3.0:1775764896.153021:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.153033:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98131968, count: 4096 00000080:00200000:3.0:1775764896.153039:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98131968, bytes: 4096 00000080:00200000:3.0:1775764896.153041:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.153043:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23959, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.153044:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98131968, 98136064) 0 0 00000080:00200000:3.0:1775764896.153046:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.153050:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23958, 23958] 00020000:00200000:3.0:1775764896.153053:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98131968 chunk: [98131968, 98136064) 98136064, 4096 00020000:00200000:3.0:1775764896.153054:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.153061:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23958, 23958] 00000008:00010000:3.0:1775764896.153064:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.153066:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.153072:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153078:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98131968 98136063) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153084:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153088:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153092:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.153096:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98131968, 98136064) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.153100:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23958 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.153125:0:18302:0:(rw26.c:858:ll_write_end()) pos 98131968, len 4096, copied 4096 00000080:00200000:3.0:1775764896.153130:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000eec8f161[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.153133:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b7a8eec6 17ffffc0001001 3:0 ffff90c212819d20 23958 lru 00000080:00200000:3.0:1775764896.153138:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c3e8b254 23958: 1< 0 - - > 2< 98131968 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2482176 - | + - + - > 5< + - + - | 0 - | 3478 - -> 00000080:00200000:3.0:1775764896.153140:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000eec8f161 00000080:00200000:3.0:1775764896.153141:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.153144:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.153152:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.153158:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.153165:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.153172:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.153184:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.153187:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.153189:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.153198:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.153201:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.153203:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.153207:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153215:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.153223:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.153225:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.153229:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.153232:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23958, 23958] 00000080:00200000:3.0:1775764896.153241:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.153244:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.153272:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98136064 i_blocks 90112 00000080:00200000:3.0:1775764896.153276:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.153291:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98136064, count: 4096 00000080:00200000:3.0:1775764896.153301:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98136064, bytes: 4096 00000080:00200000:3.0:1775764896.153303:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.153306:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23960, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.153308:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98136064, 98140160) 0 0 00000080:00200000:3.0:1775764896.153310:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.153315:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23959, 23959] 00020000:00200000:3.0:1775764896.153318:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98136064 chunk: [98136064, 98140160) 98140160, 4096 00020000:00200000:3.0:1775764896.153320:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.153331:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23959, 23959] 00000008:00010000:3.0:1775764896.153337:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.153339:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.153347:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153355:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98136064 98140159) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153363:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153376:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153383:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.153391:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98136064, 98140160) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.153395:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23959 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.153414:0:18302:0:(rw26.c:858:ll_write_end()) pos 98136064, len 4096, copied 4096 00000080:00200000:3.0:1775764896.153418:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000dd1b6f82[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.153421:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000acaf001b 17ffffc0001001 3:0 ffff90c212819380 23959 lru 00000080:00200000:3.0:1775764896.153426:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006ed3cc80 23959: 1< 0 - - > 2< 98136064 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2478080 - | + - + - > 5< + - + - | 0 - | 3479 - -> 00000080:00200000:3.0:1775764896.153428:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000dd1b6f82 00000080:00200000:3.0:1775764896.153429:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.153431:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.153533:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.153542:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.153549:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.153557:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.153575:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.153578:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.153581:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.153592:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.153594:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.153597:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.153601:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153610:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.153619:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.153621:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.153626:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.153628:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23959, 23959] 00000080:00200000:3.0:1775764896.153640:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.153643:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.153673:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98140160 i_blocks 90112 00000080:00200000:3.0:1775764896.153677:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.153693:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98140160, count: 4096 00000080:00200000:3.0:1775764896.153703:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98140160, bytes: 4096 00000080:00200000:3.0:1775764896.153706:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.153709:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23961, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.153711:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98140160, 98144256) 0 0 00000080:00200000:3.0:1775764896.153713:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.153717:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23960, 23960] 00020000:00200000:3.0:1775764896.153721:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98140160 chunk: [98140160, 98144256) 98144256, 4096 00020000:00200000:3.0:1775764896.153723:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.153733:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23960, 23960] 00000008:00010000:3.0:1775764896.153739:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.153741:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.153749:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153757:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98140160 98144255) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153766:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153773:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153780:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.153787:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98140160, 98144256) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.153793:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23960 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.153811:0:18302:0:(rw26.c:858:ll_write_end()) pos 98140160, len 4096, copied 4096 00000080:00200000:3.0:1775764896.153815:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cf19ea8d[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.153818:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005465d7dc 17ffffc0001001 3:0 ffff90c212819620 23960 lru 00000080:00200000:3.0:1775764896.153824:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b0b430b5 23960: 1< 0 - - > 2< 98140160 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2473984 - | + - + - > 5< + - + - | 0 - | 3480 - -> 00000080:00200000:3.0:1775764896.153826:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cf19ea8d 00000080:00200000:3.0:1775764896.153827:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.153830:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.153838:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.153846:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.153854:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.153861:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.153900:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.153904:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.153906:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.153916:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.153918:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.153921:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.153925:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.153933:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.153941:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.153944:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.153948:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.153951:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23960, 23960] 00000080:00200000:3.0:1775764896.153962:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.153965:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.153996:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98144256 i_blocks 90112 00000080:00200000:3.0:1775764896.154000:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.154016:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98144256, count: 4096 00000080:00200000:3.0:1775764896.154027:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98144256, bytes: 4096 00000080:00200000:3.0:1775764896.154029:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.154033:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23962, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.154035:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98144256, 98148352) 0 0 00000080:00200000:3.0:1775764896.154037:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.154043:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23961, 23961] 00020000:00200000:3.0:1775764896.154046:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98144256 chunk: [98144256, 98148352) 98148352, 4096 00020000:00200000:3.0:1775764896.154048:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.154060:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23961, 23961] 00000008:00010000:3.0:1775764896.154066:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.154068:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.154076:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154085:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98144256 98148351) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154093:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154101:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154109:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.154130:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98144256, 98148352) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.154136:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23961 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.154152:0:18302:0:(rw26.c:858:ll_write_end()) pos 98144256, len 4096, copied 4096 00000080:00200000:3.0:1775764896.154156:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000531a2fae[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.154159:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000665dc5e9 17ffffc0001001 3:0 ffff90c212819e00 23961 lru 00000080:00200000:3.0:1775764896.154164:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000010d7f575 23961: 1< 0 - - > 2< 98144256 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2469888 - | + - + - > 5< + - + - | 0 - | 3481 - -> 00000080:00200000:3.0:1775764896.154166:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000531a2fae 00000080:00200000:3.0:1775764896.154167:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.154169:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.154178:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.154186:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.154193:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.154201:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.154213:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.154216:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.154219:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.154229:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.154232:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.154234:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.154239:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154247:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.154255:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.154257:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.154262:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.154265:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23961, 23961] 00000080:00200000:3.0:1775764896.154276:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.154279:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.154311:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98148352 i_blocks 90112 00000080:00200000:3.0:1775764896.154315:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.154332:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98148352, count: 4096 00000080:00200000:3.0:1775764896.154343:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98148352, bytes: 4096 00000080:00200000:3.0:1775764896.154346:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.154349:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23963, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.154350:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98148352, 98152448) 0 0 00000080:00200000:3.0:1775764896.154352:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.154357:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23962, 23962] 00020000:00200000:3.0:1775764896.154360:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98148352 chunk: [98148352, 98152448) 98152448, 4096 00020000:00200000:3.0:1775764896.154363:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.154373:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23962, 23962] 00000008:00010000:3.0:1775764896.154378:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.154380:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.154388:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154397:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98148352 98152447) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154407:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154414:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154420:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.154429:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98148352, 98152448) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.154434:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23962 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.154541:0:18302:0:(rw26.c:858:ll_write_end()) pos 98148352, len 4096, copied 4096 00000080:00200000:3.0:1775764896.154546:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e31be80e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.154549:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000016fe2e11 17ffffc0001001 3:0 ffff90c212819540 23962 lru 00000080:00200000:3.0:1775764896.154555:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000015b77caa 23962: 1< 0 - - > 2< 98148352 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2465792 - | + - + - > 5< + - + - | 0 - | 3482 - -> 00000080:00200000:3.0:1775764896.154557:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e31be80e 00000080:00200000:3.0:1775764896.154558:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.154560:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.154568:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.154576:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.154583:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.154596:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.154608:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.154612:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.154614:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.154624:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.154627:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.154630:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.154634:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154643:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.154651:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.154653:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.154657:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.154660:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23962, 23962] 00000080:00200000:3.0:1775764896.154671:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.154674:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.154705:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98152448 i_blocks 90112 00000080:00200000:3.0:1775764896.154709:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.154726:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98152448, count: 4096 00000080:00200000:3.0:1775764896.154737:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98152448, bytes: 4096 00000080:00200000:3.0:1775764896.154739:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.154743:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23964, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.154745:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98152448, 98156544) 0 0 00000080:00200000:3.0:1775764896.154747:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.154752:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23963, 23963] 00020000:00200000:3.0:1775764896.154755:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98152448 chunk: [98152448, 98156544) 98156544, 4096 00020000:00200000:3.0:1775764896.154757:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.154767:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23963, 23963] 00000008:00010000:3.0:1775764896.154771:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.154774:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.154781:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154789:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98152448 98156543) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154797:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154804:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154810:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.154818:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98152448, 98156544) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.154824:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23963 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.154852:0:18302:0:(rw26.c:858:ll_write_end()) pos 98152448, len 4096, copied 4096 00000080:00200000:3.0:1775764896.154857:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000076134a62[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.154859:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006838c9f1 17ffffc0001001 3:0 ffff90c21281a9a0 23963 lru 00000080:00200000:3.0:1775764896.154865:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006319f5a3 23963: 1< 0 - - > 2< 98152448 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2461696 - | + - + - > 5< + - + - | 0 - | 3483 - -> 00000080:00200000:3.0:1775764896.154868:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000076134a62 00000080:00200000:3.0:1775764896.154893:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.154895:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.154904:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.154912:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.154920:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.154927:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.154938:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.154941:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.154943:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.154953:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.154956:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.154958:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.154962:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.154970:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.154978:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.154981:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.154985:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.154988:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23963, 23963] 00000080:00200000:3.0:1775764896.154998:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.155001:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.155029:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98156544 i_blocks 90112 00000080:00200000:3.0:1775764896.155032:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.155048:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98156544, count: 4096 00000080:00200000:3.0:1775764896.155059:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98156544, bytes: 4096 00000080:00200000:3.0:1775764896.155062:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.155065:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23965, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.155067:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98156544, 98160640) 0 0 00000080:00200000:3.0:1775764896.155069:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.155075:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23964, 23964] 00020000:00200000:3.0:1775764896.155078:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98156544 chunk: [98156544, 98160640) 98160640, 4096 00020000:00200000:3.0:1775764896.155080:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.155092:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23964, 23964] 00000008:00010000:3.0:1775764896.155097:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.155099:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.155107:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155129:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98156544 98160639) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155138:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155145:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155158:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.155167:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98156544, 98160640) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.155173:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23964 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.155190:0:18302:0:(rw26.c:858:ll_write_end()) pos 98156544, len 4096, copied 4096 00000080:00200000:3.0:1775764896.155194:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000000e06c03[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.155197:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c2e0afb7 17ffffc0001001 3:0 ffff90c21281a2a0 23964 lru 00000080:00200000:3.0:1775764896.155203:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000075bc79c5 23964: 1< 0 - - > 2< 98156544 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2457600 - | + - + - > 5< + - + - | 0 - | 3484 - -> 00000080:00200000:3.0:1775764896.155205:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000000e06c03 00000080:00200000:3.0:1775764896.155206:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.155209:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.155216:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.155224:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.155231:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.155239:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.155250:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.155253:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.155256:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.155267:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.155270:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.155272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.155276:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155284:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.155293:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.155295:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.155299:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.155303:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23964, 23964] 00000080:00200000:3.0:1775764896.155314:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.155317:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.155347:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98160640 i_blocks 90112 00000080:00200000:3.0:1775764896.155350:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.155366:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98160640, count: 4096 00000080:00200000:3.0:1775764896.155375:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98160640, bytes: 4096 00000080:00200000:3.0:1775764896.155378:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.155382:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23966, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.155383:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98160640, 98164736) 0 0 00000080:00200000:3.0:1775764896.155385:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.155391:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23965, 23965] 00020000:00200000:3.0:1775764896.155394:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98160640 chunk: [98160640, 98164736) 98164736, 4096 00020000:00200000:3.0:1775764896.155396:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.155408:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23965, 23965] 00000008:00010000:3.0:1775764896.155413:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.155415:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.155424:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155433:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98160640 98164735) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155532:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155540:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155547:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.155556:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98160640, 98164736) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.155562:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23965 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.155582:0:18302:0:(rw26.c:858:ll_write_end()) pos 98160640, len 4096, copied 4096 00000080:00200000:3.0:1775764896.155586:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000acdd3e42[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.155589:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e056c7c9 17ffffc0001001 3:0 ffff90c21281ad20 23965 lru 00000080:00200000:3.0:1775764896.155595:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ce5d66f2 23965: 1< 0 - - > 2< 98160640 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2453504 - | + - + - > 5< + - + - | 0 - | 3485 - -> 00000080:00200000:3.0:1775764896.155597:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000acdd3e42 00000080:00200000:3.0:1775764896.155598:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.155601:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.155609:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.155617:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.155624:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.155631:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.155643:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.155645:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.155647:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.155656:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.155659:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.155661:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.155665:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155673:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.155681:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.155683:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.155686:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.155689:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23965, 23965] 00000080:00200000:3.0:1775764896.155699:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.155702:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.155731:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98164736 i_blocks 90112 00000080:00200000:3.0:1775764896.155734:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.155751:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98164736, count: 4096 00000080:00200000:3.0:1775764896.155761:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98164736, bytes: 4096 00000080:00200000:3.0:1775764896.155763:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.155767:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23967, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.155769:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98164736, 98168832) 0 0 00000080:00200000:3.0:1775764896.155771:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.155776:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23966, 23966] 00020000:00200000:3.0:1775764896.155779:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98164736 chunk: [98164736, 98168832) 98168832, 4096 00020000:00200000:3.0:1775764896.155781:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.155792:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23966, 23966] 00000008:00010000:3.0:1775764896.155797:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.155800:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.155807:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155817:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98164736 98168831) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155825:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155833:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155840:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.155848:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98164736, 98168832) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.155853:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23966 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.155898:0:18302:0:(rw26.c:858:ll_write_end()) pos 98164736, len 4096, copied 4096 00000080:00200000:3.0:1775764896.155903:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007e5bf8f7[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.155905:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e2c00fea 17ffffc0001021 2:0 ffff90c21281a380 23966 lru 00000080:00200000:3.0:1775764896.155911:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000033e8c008 23966: 1< 0 - - > 2< 98164736 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2449408 - | + - + - > 5< + - + - | 0 - | 3486 - -> 00000080:00200000:3.0:1775764896.155913:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007e5bf8f7 00000080:00200000:3.0:1775764896.155914:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.155916:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.155924:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.155932:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.155939:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.155946:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.155957:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.155960:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.155962:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.155971:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.155974:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.155977:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.155981:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.155988:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.155997:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.156000:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.156004:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.156007:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23966, 23966] 00000080:00200000:3.0:1775764896.156017:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.156021:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.156050:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98168832 i_blocks 90112 00000080:00200000:3.0:1775764896.156054:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.156069:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98168832, count: 4096 00000080:00200000:3.0:1775764896.156079:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98168832, bytes: 4096 00000080:00200000:3.0:1775764896.156081:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.156085:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23968, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.156087:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98168832, 98172928) 0 0 00000080:00200000:3.0:1775764896.156089:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.156094:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23967, 23967] 00020000:00200000:3.0:1775764896.156097:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98168832 chunk: [98168832, 98172928) 98172928, 4096 00020000:00200000:3.0:1775764896.156100:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.156110:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23967, 23967] 00000008:00010000:3.0:1775764896.156128:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.156131:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.156139:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156148:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98168832 98172927) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156156:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156168:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156174:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.156182:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98168832, 98172928) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.156187:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23967 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.156202:0:18302:0:(rw26.c:858:ll_write_end()) pos 98168832, len 4096, copied 4096 00000080:00200000:3.0:1775764896.156206:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a5bbc087[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.156209:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ae76f09e 17ffffc0001001 3:0 ffff90c21281a620 23967 lru 00000080:00200000:3.0:1775764896.156215:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a3dd7385 23967: 1< 0 - - > 2< 98168832 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2445312 - | + - + - > 5< + - + - | 0 - | 3487 - -> 00000080:00200000:3.0:1775764896.156217:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a5bbc087 00000080:00200000:3.0:1775764896.156218:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.156220:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.156227:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.156235:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.156242:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.156249:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.156265:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.156268:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.156271:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.156280:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.156283:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.156286:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.156289:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156298:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.156306:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.156308:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.156312:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.156316:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23967, 23967] 00000080:00200000:3.0:1775764896.156326:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.156329:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.156358:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98172928 i_blocks 90112 00000080:00200000:3.0:1775764896.156362:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.156378:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98172928, count: 4096 00000080:00200000:3.0:1775764896.156388:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98172928, bytes: 4096 00000080:00200000:3.0:1775764896.156391:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.156395:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23969, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.156396:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98172928, 98177024) 0 0 00000080:00200000:3.0:1775764896.156399:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.156404:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23968, 23968] 00020000:00200000:3.0:1775764896.156407:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98172928 chunk: [98172928, 98177024) 98177024, 4096 00020000:00200000:3.0:1775764896.156410:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.156422:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23968, 23968] 00000008:00010000:3.0:1775764896.156427:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.156430:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.156521:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156532:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98172928 98177023) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156541:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156549:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156555:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.156564:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98172928, 98177024) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.156570:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23968 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.156588:0:18302:0:(rw26.c:858:ll_write_end()) pos 98172928, len 4096, copied 4096 00000080:00200000:3.0:1775764896.156593:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b3c10bee[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.156595:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000659e1884 17ffffc0001001 3:0 ffff90c21281ae00 23968 lru 00000080:00200000:3.0:1775764896.156602:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000011e13315 23968: 1< 0 - - > 2< 98172928 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2441216 - | + - + - > 5< + - + - | 0 - | 3488 - -> 00000080:00200000:3.0:1775764896.156604:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b3c10bee 00000080:00200000:3.0:1775764896.156605:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.156607:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.156616:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.156623:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.156630:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.156638:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.156650:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.156653:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.156656:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.156667:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.156670:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.156673:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.156677:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156686:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.156694:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.156697:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.156702:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.156705:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23968, 23968] 00000080:00200000:3.0:1775764896.156717:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.156720:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.156751:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98177024 i_blocks 90112 00000080:00200000:3.0:1775764896.156755:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.156771:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98177024, count: 4096 00000080:00200000:3.0:1775764896.156782:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98177024, bytes: 4096 00000080:00200000:3.0:1775764896.156786:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.156789:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23970, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.156791:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98177024, 98181120) 0 0 00000080:00200000:3.0:1775764896.156793:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.156798:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23969, 23969] 00020000:00200000:3.0:1775764896.156801:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98177024 chunk: [98177024, 98181120) 98181120, 4096 00020000:00200000:3.0:1775764896.156804:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.156815:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23969, 23969] 00000008:00010000:3.0:1775764896.156821:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.156824:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.156832:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156841:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98177024 98181119) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156849:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156856:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.156864:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.156902:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98177024, 98181120) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.156907:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23969 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.156924:0:18302:0:(rw26.c:858:ll_write_end()) pos 98177024, len 4096, copied 4096 00000080:00200000:3.0:1775764896.156928:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a26247f2[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.156931:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006bc6ac2c 17ffffc0001001 3:0 ffff90c21281a540 23969 lru 00000080:00200000:3.0:1775764896.156937:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d1784211 23969: 1< 0 - - > 2< 98177024 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2437120 - | + - + - > 5< + - + - | 0 - | 3489 - -> 00000080:00200000:3.0:1775764896.156939:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a26247f2 00000080:00200000:3.0:1775764896.156940:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.156943:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.156950:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.156958:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.156964:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.156972:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.156983:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.156986:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.156988:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.156998:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.157001:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.157003:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.157008:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157016:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.157024:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.157026:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.157030:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.157032:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23969, 23969] 00000080:00200000:3.0:1775764896.157042:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.157045:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.157071:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98181120 i_blocks 90112 00000080:00200000:3.0:1775764896.157075:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.157089:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98181120, count: 4096 00000080:00200000:3.0:1775764896.157098:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98181120, bytes: 4096 00000080:00200000:3.0:1775764896.157101:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.157104:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23971, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.157106:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98181120, 98185216) 0 0 00000080:00200000:3.0:1775764896.157108:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.157112:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23970, 23970] 00020000:00200000:3.0:1775764896.157129:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98181120 chunk: [98181120, 98185216) 98185216, 4096 00020000:00200000:3.0:1775764896.157131:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.157143:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23970, 23970] 00000008:00010000:3.0:1775764896.157148:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.157150:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.157158:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157168:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98181120 98185215) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157176:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157183:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157191:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.157198:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98181120, 98185216) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.157204:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23970 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.157220:0:18302:0:(rw26.c:858:ll_write_end()) pos 98181120, len 4096, copied 4096 00000080:00200000:3.0:1775764896.157225:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000d19477f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.157228:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000d446c57 17ffffc0001001 3:0 ffff90c21281a460 23970 lru 00000080:00200000:3.0:1775764896.157234:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000591ee16d 23970: 1< 0 - - > 2< 98181120 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2433024 - | + - + - > 5< + - + - | 0 - | 3490 - -> 00000080:00200000:3.0:1775764896.157236:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000d19477f 00000080:00200000:3.0:1775764896.157237:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.157239:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.157247:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.157255:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.157262:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.157274:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.157286:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.157289:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.157291:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.157300:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.157303:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.157306:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.157309:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157317:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.157325:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.157327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.157331:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.157333:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23970, 23970] 00000080:00200000:3.0:1775764896.157344:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.157347:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.157376:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98185216 i_blocks 90112 00000080:00200000:3.0:1775764896.157379:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.157395:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98185216, count: 4096 00000080:00200000:3.0:1775764896.157405:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98185216, bytes: 4096 00000080:00200000:3.0:1775764896.157408:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.157413:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23972, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.157415:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98185216, 98189312) 0 0 00000080:00200000:3.0:1775764896.157417:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.157423:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23971, 23971] 00020000:00200000:3.0:1775764896.157426:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98185216 chunk: [98185216, 98189312) 98189312, 4096 00020000:00200000:3.0:1775764896.157428:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.157530:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23971, 23971] 00000008:00010000:3.0:1775764896.157537:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.157540:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.157548:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157557:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98185216 98189311) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157566:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157573:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157580:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.157587:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98185216, 98189312) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.157592:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23971 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.157611:0:18302:0:(rw26.c:858:ll_write_end()) pos 98185216, len 4096, copied 4096 00000080:00200000:3.0:1775764896.157615:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000142acd32[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.157618:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000237287ab 17ffffc0001001 3:0 ffff90c21281aa80 23971 lru 00000080:00200000:3.0:1775764896.157623:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008d7ff524 23971: 1< 0 - - > 2< 98185216 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2428928 - | + - + - > 5< + - + - | 0 - | 3491 - -> 00000080:00200000:3.0:1775764896.157626:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000142acd32 00000080:00200000:3.0:1775764896.157627:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.157629:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.157636:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.157644:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.157652:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.157659:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.157670:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.157673:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.157675:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.157686:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.157689:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.157691:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.157695:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157703:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.157710:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.157713:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.157716:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.157719:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23971, 23971] 00000080:00200000:3.0:1775764896.157729:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.157732:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.157761:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98189312 i_blocks 90112 00000080:00200000:3.0:1775764896.157765:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.157779:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98189312, count: 4096 00000080:00200000:3.0:1775764896.157788:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98189312, bytes: 4096 00000080:00200000:3.0:1775764896.157790:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.157793:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23973, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.157795:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98189312, 98193408) 0 0 00000080:00200000:3.0:1775764896.157797:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.157802:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23972, 23972] 00020000:00200000:3.0:1775764896.157804:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98189312 chunk: [98189312, 98193408) 98193408, 4096 00020000:00200000:3.0:1775764896.157806:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.157815:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23972, 23972] 00000008:00010000:3.0:1775764896.157820:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.157822:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.157829:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157837:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98189312 98193407) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157844:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157851:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157862:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.157892:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98189312, 98193408) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.157898:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23972 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.157912:0:18302:0:(rw26.c:858:ll_write_end()) pos 98189312, len 4096, copied 4096 00000080:00200000:3.0:1775764896.157916:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005b1150aa[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.157918:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000087f51a34 17ffffc0001001 3:0 ffff90c21281a700 23972 lru 00000080:00200000:3.0:1775764896.157924:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006c8c6e5c 23972: 1< 0 - - > 2< 98189312 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2424832 - | + - + - > 5< + - + - | 0 - | 3492 - -> 00000080:00200000:3.0:1775764896.157926:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005b1150aa 00000080:00200000:3.0:1775764896.157927:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.157929:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.157936:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.157943:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.157950:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.157957:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.157968:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.157971:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.157974:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.157984:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.157986:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.157988:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.157991:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.157998:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.158006:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.158009:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.158013:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.158015:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23972, 23972] 00000080:00200000:3.0:1775764896.158026:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.158029:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.158058:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98193408 i_blocks 90112 00000080:00200000:3.0:1775764896.158062:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.158077:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98193408, count: 4096 00000080:00200000:3.0:1775764896.158086:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98193408, bytes: 4096 00000080:00200000:3.0:1775764896.158088:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.158090:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23974, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.158092:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98193408, 98197504) 0 0 00000080:00200000:3.0:1775764896.158093:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.158097:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23973, 23973] 00020000:00200000:3.0:1775764896.158099:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98193408 chunk: [98193408, 98197504) 98197504, 4096 00020000:00200000:3.0:1775764896.158101:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.158108:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23973, 23973] 00000008:00010000:3.0:1775764896.158111:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.158125:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.158133:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158143:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98193408 98197503) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158152:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158159:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158165:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.158172:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98193408, 98197504) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.158178:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23973 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.158194:0:18302:0:(rw26.c:858:ll_write_end()) pos 98193408, len 4096, copied 4096 00000080:00200000:3.0:1775764896.158198:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a06d0ef4[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.158201:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000027a46078 17ffffc0001001 3:0 ffff90c21281ac40 23973 lru 00000080:00200000:3.0:1775764896.158207:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004fa21253 23973: 1< 0 - - > 2< 98193408 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2420736 - | + - + - > 5< + - + - | 0 - | 3493 - -> 00000080:00200000:3.0:1775764896.158209:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a06d0ef4 00000080:00200000:3.0:1775764896.158210:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.158212:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.158220:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.158227:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.158233:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.158241:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.158252:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.158255:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.158257:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.158267:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.158270:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.158272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.158276:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158284:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.158289:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.158291:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.158294:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.158296:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23973, 23973] 00000080:00200000:3.0:1775764896.158303:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.158305:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.158325:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98197504 i_blocks 90112 00000080:00200000:3.0:1775764896.158328:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.158343:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98197504, count: 4096 00000080:00200000:3.0:1775764896.158352:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98197504, bytes: 4096 00000080:00200000:3.0:1775764896.158355:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.158359:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23975, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.158360:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98197504, 98201600) 0 0 00000080:00200000:3.0:1775764896.158362:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.158367:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23974, 23974] 00020000:00200000:3.0:1775764896.158370:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98197504 chunk: [98197504, 98201600) 98201600, 4096 00020000:00200000:3.0:1775764896.158373:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.158383:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23974, 23974] 00000008:00010000:3.0:1775764896.158388:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.158391:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.158398:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158407:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98197504 98201599) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158415:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158420:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158425:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.158430:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98197504, 98201600) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.158433:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23974 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.158518:0:18302:0:(rw26.c:858:ll_write_end()) pos 98197504, len 4096, copied 4096 00000080:00200000:3.0:1775764896.158521:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ec44ea11[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.158523:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c7604d30 17ffffc0001001 3:0 ffff90c21281a8c0 23974 lru 00000080:00200000:3.0:1775764896.158527:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b889b112 23974: 1< 0 - - > 2< 98197504 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2416640 - | + - + - > 5< + - + - | 0 - | 3494 - -> 00000080:00200000:3.0:1775764896.158528:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ec44ea11 00000080:00200000:3.0:1775764896.158529:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.158531:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.158536:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.158541:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.158546:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.158550:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.158557:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.158558:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.158560:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.158566:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.158568:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.158570:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.158572:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158578:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.158584:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.158585:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.158587:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.158589:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23974, 23974] 00000080:00200000:3.0:1775764896.158597:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.158599:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.158811:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98201600 i_blocks 90112 00000080:00200000:3.0:1775764896.158815:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.158827:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98201600, count: 4096 00000080:00200000:3.0:1775764896.158834:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98201600, bytes: 4096 00000080:00200000:3.0:1775764896.158836:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.158839:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23976, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.158840:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98201600, 98205696) 0 0 00000080:00200000:3.0:1775764896.158841:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.158846:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23975, 23975] 00020000:00200000:3.0:1775764896.158848:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98201600 chunk: [98201600, 98205696) 98205696, 4096 00020000:00200000:3.0:1775764896.158849:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.158856:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23975, 23975] 00000008:00010000:3.0:1775764896.158860:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.158862:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.158867:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158893:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98201600 98205695) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158901:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158914:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.158920:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.158928:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98201600, 98205696) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.158933:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23975 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.158954:0:18302:0:(rw26.c:858:ll_write_end()) pos 98201600, len 4096, copied 4096 00000080:00200000:3.0:1775764896.158958:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002090eb1f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.158961:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d7defe09 17ffffc0001001 3:0 ffff90c21281ab60 23975 lru 00000080:00200000:3.0:1775764896.158967:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002a1bb08f 23975: 1< 0 - - > 2< 98201600 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2412544 - | + - + - > 5< + - + - | 0 - | 3495 - -> 00000080:00200000:3.0:1775764896.158969:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002090eb1f 00000080:00200000:3.0:1775764896.158970:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.158972:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.158981:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.158988:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.158995:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.159001:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.159018:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.159021:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.159023:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.159033:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.159035:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.159038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.159043:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159051:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.159059:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.159060:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.159064:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.159066:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23975, 23975] 00000080:00200000:3.0:1775764896.159073:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.159075:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.159095:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98205696 i_blocks 90112 00000080:00200000:3.0:1775764896.159097:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.159108:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98205696, count: 4096 00000080:00200000:3.0:1775764896.159131:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98205696, bytes: 4096 00000080:00200000:3.0:1775764896.159133:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.159136:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23977, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.159137:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98205696, 98209792) 0 0 00000080:00200000:3.0:1775764896.159138:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.159142:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23976, 23976] 00020000:00200000:3.0:1775764896.159145:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98205696 chunk: [98205696, 98209792) 98209792, 4096 00020000:00200000:3.0:1775764896.159146:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.159154:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23976, 23976] 00000008:00010000:3.0:1775764896.159157:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.159159:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.159164:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159172:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98205696 98209791) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159181:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159188:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159195:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.159203:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98205696, 98209792) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.159208:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23976 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.159225:0:18302:0:(rw26.c:858:ll_write_end()) pos 98205696, len 4096, copied 4096 00000080:00200000:3.0:1775764896.159229:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000007974f32[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.159232:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005c301837 17ffffc0001001 3:0 ffff90c21281a000 23976 lru 00000080:00200000:3.0:1775764896.159237:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006286be7a 23976: 1< 0 - - > 2< 98205696 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2408448 - | + - + - > 5< + - + - | 0 - | 3496 - -> 00000080:00200000:3.0:1775764896.159239:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000007974f32 00000080:00200000:3.0:1775764896.159240:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.159243:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.159251:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.159260:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.159267:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.159275:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.159287:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.159290:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.159292:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.159303:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.159305:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.159308:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.159334:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159342:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.159357:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.159360:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.159364:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.159367:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23976, 23976] 00000080:00200000:3.0:1775764896.159379:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.159382:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.159412:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98209792 i_blocks 90112 00000080:00200000:3.0:1775764896.159415:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.159431:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98209792, count: 4096 00000080:00200000:3.0:1775764896.159530:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98209792, bytes: 4096 00000080:00200000:3.0:1775764896.159533:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.159537:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23978, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.159539:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98209792, 98213888) 0 0 00000080:00200000:3.0:1775764896.159541:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.159547:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23977, 23977] 00020000:00200000:3.0:1775764896.159550:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98209792 chunk: [98209792, 98213888) 98213888, 4096 00020000:00200000:3.0:1775764896.159552:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.159564:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23977, 23977] 00000008:00010000:3.0:1775764896.159570:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.159572:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.159580:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159589:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98209792 98213887) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159598:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159605:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159613:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.159620:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98209792, 98213888) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.159625:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23977 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.159644:0:18302:0:(rw26.c:858:ll_write_end()) pos 98209792, len 4096, copied 4096 00000080:00200000:3.0:1775764896.159648:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000071723ddf[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.159651:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d3b3e71b 17ffffc0001001 3:0 ffff90c21281aee0 23977 lru 00000080:00200000:3.0:1775764896.159656:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a8855231 23977: 1< 0 - - > 2< 98209792 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2404352 - | + - + - > 5< + - + - | 0 - | 3497 - -> 00000080:00200000:3.0:1775764896.159658:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000071723ddf 00000080:00200000:3.0:1775764896.159660:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.159662:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.159670:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.159678:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.159685:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.159692:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.159704:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.159707:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.159709:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.159720:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.159722:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.159725:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.159729:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159737:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.159745:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.159747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.159751:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.159754:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23977, 23977] 00000080:00200000:3.0:1775764896.159765:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.159769:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.159798:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98213888 i_blocks 90112 00000080:00200000:3.0:1775764896.159801:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.159817:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98213888, count: 4096 00000080:00200000:3.0:1775764896.159828:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98213888, bytes: 4096 00000080:00200000:3.0:1775764896.159830:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.159834:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23979, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.159835:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98213888, 98217984) 0 0 00000080:00200000:3.0:1775764896.159838:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.159843:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23978, 23978] 00020000:00200000:3.0:1775764896.159846:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98213888 chunk: [98213888, 98217984) 98217984, 4096 00020000:00200000:3.0:1775764896.159848:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.159857:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23978, 23978] 00000008:00010000:3.0:1775764896.159862:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.159865:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.159892:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159901:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98213888 98217983) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159911:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159918:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.159924:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.159932:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98213888, 98217984) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.159938:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23978 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.159954:0:18302:0:(rw26.c:858:ll_write_end()) pos 98213888, len 4096, copied 4096 00000080:00200000:3.0:1775764896.159958:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000025937d7e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.159961:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b36e272b 17ffffc0001001 3:0 ffff90c21281a0e0 23978 lru 00000080:00200000:3.0:1775764896.159967:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000dce0a80b 23978: 1< 0 - - > 2< 98213888 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2400256 - | + - + - > 5< + - + - | 0 - | 3498 - -> 00000080:00200000:3.0:1775764896.159969:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000025937d7e 00000080:00200000:3.0:1775764896.159969:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.159971:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.159979:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.159987:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.159994:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.160007:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.160018:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.160021:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.160024:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.160033:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.160036:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.160038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.160042:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160049:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.160058:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.160060:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.160065:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.160067:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23978, 23978] 00000080:00200000:3.0:1775764896.160078:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.160081:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.160111:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98217984 i_blocks 90112 00000080:00200000:3.0:1775764896.160130:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.160147:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98217984, count: 4096 00000080:00200000:3.0:1775764896.160158:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98217984, bytes: 4096 00000080:00200000:3.0:1775764896.160161:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.160165:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23980, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.160166:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98217984, 98222080) 0 0 00000080:00200000:3.0:1775764896.160169:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.160174:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23979, 23979] 00020000:00200000:3.0:1775764896.160177:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98217984 chunk: [98217984, 98222080) 98222080, 4096 00020000:00200000:3.0:1775764896.160180:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.160191:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23979, 23979] 00000008:00010000:3.0:1775764896.160196:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.160198:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.160205:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160214:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98217984 98222079) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160222:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160228:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160235:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.160243:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98217984, 98222080) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.160248:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23979 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.160264:0:18302:0:(rw26.c:858:ll_write_end()) pos 98217984, len 4096, copied 4096 00000080:00200000:3.0:1775764896.160268:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004f7f77ef[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.160271:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000015a7f1a0 17ffffc0001001 3:0 ffff90c21281a1c0 23979 lru 00000080:00200000:3.0:1775764896.160276:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000092460793 23979: 1< 0 - - > 2< 98217984 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2396160 - | + - + - > 5< + - + - | 0 - | 3499 - -> 00000080:00200000:3.0:1775764896.160279:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004f7f77ef 00000080:00200000:3.0:1775764896.160280:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.160282:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.160289:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.160296:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.160303:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.160309:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.160321:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.160324:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.160326:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.160335:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.160338:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.160340:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.160343:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160352:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.160360:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.160364:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.160368:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.160370:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23979, 23979] 00000080:00200000:3.0:1775764896.160381:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.160384:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.160802:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98222080 i_blocks 90112 00000080:00200000:3.0:1775764896.160807:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.160828:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98222080, count: 4096 00000080:00200000:3.0:1775764896.160841:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98222080, bytes: 4096 00000080:00200000:3.0:1775764896.160844:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.160848:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23981, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.160850:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98222080, 98226176) 0 0 00000080:00200000:3.0:1775764896.160853:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.160859:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23980, 23980] 00020000:00200000:3.0:1775764896.160862:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98222080 chunk: [98222080, 98226176) 98226176, 4096 00020000:00200000:3.0:1775764896.160865:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.160904:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23980, 23980] 00000008:00010000:3.0:1775764896.160911:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.160914:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.160923:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160932:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98222080 98226175) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160941:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160949:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.160961:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.160970:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98222080, 98226176) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.160975:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23980 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.160996:0:18302:0:(rw26.c:858:ll_write_end()) pos 98222080, len 4096, copied 4096 00000080:00200000:3.0:1775764896.161000:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000033abd442[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.161003:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001a211a59 17ffffc0001001 3:0 ffff90c21281a7e0 23980 lru 00000080:00200000:3.0:1775764896.161009:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a32f8bab 23980: 1< 0 - - > 2< 98222080 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2392064 - | + - + - > 5< + - + - | 0 - | 3500 - -> 00000080:00200000:3.0:1775764896.161010:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000033abd442 00000080:00200000:3.0:1775764896.161012:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.161014:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.161023:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.161030:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.161038:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.161046:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.161058:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.161061:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.161064:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.161075:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.161078:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.161081:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.161085:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161093:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.161101:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.161104:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.161108:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.161111:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23980, 23980] 00000080:00200000:3.0:1775764896.161135:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.161139:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.161170:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98226176 i_blocks 90112 00000080:00200000:3.0:1775764896.161174:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.161190:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98226176, count: 4096 00000080:00200000:3.0:1775764896.161201:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98226176, bytes: 4096 00000080:00200000:3.0:1775764896.161203:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.161207:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23982, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.161208:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98226176, 98230272) 0 0 00000080:00200000:3.0:1775764896.161210:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.161216:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23981, 23981] 00020000:00200000:3.0:1775764896.161219:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98226176 chunk: [98226176, 98230272) 98230272, 4096 00020000:00200000:3.0:1775764896.161221:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.161233:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23981, 23981] 00000008:00010000:3.0:1775764896.161238:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.161241:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.161249:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161258:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98226176 98230271) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161266:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161274:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161281:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.161289:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98226176, 98230272) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.161294:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23981 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.161330:0:18302:0:(rw26.c:858:ll_write_end()) pos 98226176, len 4096, copied 4096 00000080:00200000:3.0:1775764896.161335:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005a27b505[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.161338:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ffe13716 17ffffc0001021 2:0 ffff90c21281b0e0 23981 lru 00000080:00200000:3.0:1775764896.161343:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007567c3a5 23981: 1< 0 - - > 2< 98226176 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2387968 - | + - + - > 5< + - + - | 0 - | 3501 - -> 00000080:00200000:3.0:1775764896.161345:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005a27b505 00000080:00200000:3.0:1775764896.161346:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.161349:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.161357:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.161364:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.161371:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.161379:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.161391:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.161394:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.161396:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.161406:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.161409:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.161411:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.161415:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161423:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.161430:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.161432:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.161436:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.161518:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23981, 23981] 00000080:00200000:3.0:1775764896.161531:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.161535:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.161566:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98230272 i_blocks 90112 00000080:00200000:3.0:1775764896.161570:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.161587:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98230272, count: 4096 00000080:00200000:3.0:1775764896.161598:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98230272, bytes: 4096 00000080:00200000:3.0:1775764896.161601:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.161605:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23983, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.161606:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98230272, 98234368) 0 0 00000080:00200000:3.0:1775764896.161609:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.161615:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23982, 23982] 00020000:00200000:3.0:1775764896.161618:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98230272 chunk: [98230272, 98234368) 98234368, 4096 00020000:00200000:3.0:1775764896.161620:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.161632:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23982, 23982] 00000008:00010000:3.0:1775764896.161638:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.161641:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.161650:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161659:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98230272 98234367) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161668:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161675:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161682:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.161690:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98230272, 98234368) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.161695:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23982 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.161714:0:18302:0:(rw26.c:858:ll_write_end()) pos 98230272, len 4096, copied 4096 00000080:00200000:3.0:1775764896.161718:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000044233a27[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.161721:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e708aa88 17ffffc0001001 3:0 ffff90c21281b1c0 23982 lru 00000080:00200000:3.0:1775764896.161727:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000fe78ed71 23982: 1< 0 - - > 2< 98230272 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2383872 - | + - + - > 5< + - + - | 0 - | 3502 - -> 00000080:00200000:3.0:1775764896.161729:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000044233a27 00000080:00200000:3.0:1775764896.161730:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.161732:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.161741:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.161749:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.161756:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.161764:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.161776:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.161779:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.161781:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.161791:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.161794:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.161796:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.161801:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161809:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.161818:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.161820:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.161824:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.161827:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23982, 23982] 00000080:00200000:3.0:1775764896.161839:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.161843:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.161896:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98234368 i_blocks 90112 00000080:00200000:3.0:1775764896.161900:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.161917:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98234368, count: 4096 00000080:00200000:3.0:1775764896.161928:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98234368, bytes: 4096 00000080:00200000:3.0:1775764896.161930:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.161934:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23984, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.161936:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98234368, 98238464) 0 0 00000080:00200000:3.0:1775764896.161938:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.161944:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23983, 23983] 00020000:00200000:3.0:1775764896.161947:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98234368 chunk: [98234368, 98238464) 98238464, 4096 00020000:00200000:3.0:1775764896.161949:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.161961:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23983, 23983] 00000008:00010000:3.0:1775764896.161966:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.161968:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.161977:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161986:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98234368 98238463) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.161994:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162007:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162014:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.162022:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98234368, 98238464) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.162027:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23983 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.162048:0:18302:0:(rw26.c:858:ll_write_end()) pos 98234368, len 4096, copied 4096 00000080:00200000:3.0:1775764896.162052:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cc91eded[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.162056:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e435acf2 17ffffc0001001 3:0 ffff90c21281b7e0 23983 lru 00000080:00200000:3.0:1775764896.162061:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007d5bb4da 23983: 1< 0 - - > 2< 98234368 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2379776 - | + - + - > 5< + - + - | 0 - | 3503 - -> 00000080:00200000:3.0:1775764896.162064:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cc91eded 00000080:00200000:3.0:1775764896.162065:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.162067:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.162076:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.162084:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.162091:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.162099:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.162129:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.162133:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.162135:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.162146:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.162149:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.162151:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.162155:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162164:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.162172:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.162175:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.162179:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.162182:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23983, 23983] 00000080:00200000:3.0:1775764896.162193:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.162196:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.162225:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98238464 i_blocks 90112 00000080:00200000:3.0:1775764896.162229:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.162244:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98238464, count: 4096 00000080:00200000:3.0:1775764896.162255:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98238464, bytes: 4096 00000080:00200000:3.0:1775764896.162258:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.162262:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23985, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.162263:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98238464, 98242560) 0 0 00000080:00200000:3.0:1775764896.162265:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.162271:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23984, 23984] 00020000:00200000:3.0:1775764896.162275:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98238464 chunk: [98238464, 98242560) 98242560, 4096 00020000:00200000:3.0:1775764896.162277:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.162289:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23984, 23984] 00000008:00010000:3.0:1775764896.162295:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.162297:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.162305:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162313:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98238464 98242559) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162321:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162328:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162336:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.162343:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98238464, 98242560) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.162348:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23984 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.162364:0:18302:0:(rw26.c:858:ll_write_end()) pos 98238464, len 4096, copied 4096 00000080:00200000:3.0:1775764896.162368:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000425f7fd7[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.162371:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005b5b4c82 17ffffc0001001 3:0 ffff90c21281b9a0 23984 lru 00000080:00200000:3.0:1775764896.162377:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000789badd4 23984: 1< 0 - - > 2< 98238464 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2375680 - | + - + - > 5< + - + - | 0 - | 3504 - -> 00000080:00200000:3.0:1775764896.162379:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000425f7fd7 00000080:00200000:3.0:1775764896.162380:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.162382:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.162390:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.162399:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.162406:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.162413:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.162424:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.162427:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.162429:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.162522:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.162525:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.162529:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.162533:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162540:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.162548:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.162551:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.162555:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.162559:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23984, 23984] 00000080:00200000:3.0:1775764896.162570:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.162572:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.162601:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98242560 i_blocks 90112 00000080:00200000:3.0:1775764896.162604:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.162619:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98242560, count: 4096 00000080:00200000:3.0:1775764896.162629:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98242560, bytes: 4096 00000080:00200000:3.0:1775764896.162631:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.162635:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23986, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.162637:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98242560, 98246656) 0 0 00000080:00200000:3.0:1775764896.162639:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.162644:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23985, 23985] 00020000:00200000:3.0:1775764896.162648:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98242560 chunk: [98242560, 98246656) 98246656, 4096 00020000:00200000:3.0:1775764896.162650:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.162661:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23985, 23985] 00000008:00010000:3.0:1775764896.162666:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.162669:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.162676:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162685:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98242560 98246655) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162693:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162701:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162708:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.162717:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98242560, 98246656) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.162722:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23985 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.162740:0:18302:0:(rw26.c:858:ll_write_end()) pos 98242560, len 4096, copied 4096 00000080:00200000:3.0:1775764896.162744:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000168e30f3[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.162747:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c02341e4 17ffffc0001001 3:0 ffff90c21281b2a0 23985 lru 00000080:00200000:3.0:1775764896.162753:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000095837285 23985: 1< 0 - - > 2< 98242560 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2371584 - | + - + - > 5< + - + - | 0 - | 3505 - -> 00000080:00200000:3.0:1775764896.162756:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000168e30f3 00000080:00200000:3.0:1775764896.162757:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.162759:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.162768:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.162776:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.162783:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.162791:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.162803:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.162806:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.162808:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.162818:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.162821:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.162824:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.162828:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.162835:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.162843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.162845:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.162849:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.162852:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23985, 23985] 00000080:00200000:3.0:1775764896.162863:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.162867:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.163006:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98246656 i_blocks 90112 00000080:00200000:3.0:1775764896.163010:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.163027:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98246656, count: 4096 00000080:00200000:3.0:1775764896.163037:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98246656, bytes: 4096 00000080:00200000:3.0:1775764896.163040:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.163044:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23987, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.163045:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98246656, 98250752) 0 0 00000080:00200000:3.0:1775764896.163048:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.163053:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23986, 23986] 00020000:00200000:3.0:1775764896.163057:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98246656 chunk: [98246656, 98250752) 98250752, 4096 00020000:00200000:3.0:1775764896.163059:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.163071:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23986, 23986] 00000008:00010000:3.0:1775764896.163077:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.163079:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.163087:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163097:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98246656 98250751) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163106:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163125:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163133:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.163141:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98246656, 98250752) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.163146:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23986 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.163165:0:18302:0:(rw26.c:858:ll_write_end()) pos 98246656, len 4096, copied 4096 00000080:00200000:3.0:1775764896.163169:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006c14d8b5[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.163172:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d7e4d73c 17ffffc0001001 3:0 ffff90c21281bd20 23986 lru 00000080:00200000:3.0:1775764896.163178:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009caecdbd 23986: 1< 0 - - > 2< 98246656 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2367488 - | + - + - > 5< + - + - | 0 - | 3506 - -> 00000080:00200000:3.0:1775764896.163180:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006c14d8b5 00000080:00200000:3.0:1775764896.163181:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.163183:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.163191:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.163199:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.163206:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.163219:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.163230:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.163233:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.163235:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.163246:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.163249:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.163251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.163255:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163264:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.163272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.163274:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.163278:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.163281:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23986, 23986] 00000080:00200000:3.0:1775764896.163292:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.163295:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.163324:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98250752 i_blocks 90112 00000080:00200000:3.0:1775764896.163327:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.163342:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98250752, count: 4096 00000080:00200000:3.0:1775764896.163352:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98250752, bytes: 4096 00000080:00200000:3.0:1775764896.163355:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.163359:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23988, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.163360:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98250752, 98254848) 0 0 00000080:00200000:3.0:1775764896.163363:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.163368:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23987, 23987] 00020000:00200000:3.0:1775764896.163371:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98250752 chunk: [98250752, 98254848) 98254848, 4096 00020000:00200000:3.0:1775764896.163373:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.163384:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23987, 23987] 00000008:00010000:3.0:1775764896.163389:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.163392:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.163400:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163409:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98250752 98254847) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163417:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163424:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163431:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.163520:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98250752, 98254848) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.163527:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23987 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.163542:0:18302:0:(rw26.c:858:ll_write_end()) pos 98250752, len 4096, copied 4096 00000080:00200000:3.0:1775764896.163546:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005c5269c2[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.163549:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c0b36425 17ffffc0001001 3:0 ffff90c21281b380 23987 lru 00000080:00200000:3.0:1775764896.163554:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000543c3077 23987: 1< 0 - - > 2< 98250752 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2363392 - | + - + - > 5< + - + - | 0 - | 3507 - -> 00000080:00200000:3.0:1775764896.163557:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005c5269c2 00000080:00200000:3.0:1775764896.163558:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.163560:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.163569:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.163577:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.163584:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.163591:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.163603:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.163605:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.163608:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.163617:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.163620:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.163623:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.163626:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163634:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.163642:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.163645:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.163649:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.163652:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23987, 23987] 00000080:00200000:3.0:1775764896.163663:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.163666:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.163695:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98254848 i_blocks 90112 00000080:00200000:3.0:1775764896.163699:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.163714:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98254848, count: 4096 00000080:00200000:3.0:1775764896.163725:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98254848, bytes: 4096 00000080:00200000:3.0:1775764896.163727:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.163731:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23989, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.163733:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98254848, 98258944) 0 0 00000080:00200000:3.0:1775764896.163735:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.163741:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23988, 23988] 00020000:00200000:3.0:1775764896.163744:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98254848 chunk: [98254848, 98258944) 98258944, 4096 00020000:00200000:3.0:1775764896.163746:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.163757:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23988, 23988] 00000008:00010000:3.0:1775764896.163763:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.163765:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.163773:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163782:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98254848 98258943) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163790:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163798:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163810:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.163818:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98254848, 98258944) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.163824:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23988 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.163841:0:18302:0:(rw26.c:858:ll_write_end()) pos 98254848, len 4096, copied 4096 00000080:00200000:3.0:1775764896.163845:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000042ef2220[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.163848:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b92cbaec 17ffffc0001001 3:0 ffff90c21281b620 23988 lru 00000080:00200000:3.0:1775764896.163853:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c0718985 23988: 1< 0 - - > 2< 98254848 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2359296 - | + - + - > 5< + - + - | 0 - | 3508 - -> 00000080:00200000:3.0:1775764896.163855:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000042ef2220 00000080:00200000:3.0:1775764896.163856:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.163858:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.163866:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.163896:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.163904:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.163911:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.163923:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.163925:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.163928:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.163937:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.163940:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.163942:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.163946:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.163954:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.163963:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.163966:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.163970:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.163973:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23988, 23988] 00000080:00200000:3.0:1775764896.163984:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.163987:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.164016:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98258944 i_blocks 90112 00000080:00200000:3.0:1775764896.164020:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.164036:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98258944, count: 4096 00000080:00200000:3.0:1775764896.164047:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98258944, bytes: 4096 00000080:00200000:3.0:1775764896.164049:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.164053:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23990, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.164055:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98258944, 98263040) 0 0 00000080:00200000:3.0:1775764896.164057:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.164062:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23989, 23989] 00020000:00200000:3.0:1775764896.164065:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98258944 chunk: [98258944, 98263040) 98263040, 4096 00020000:00200000:3.0:1775764896.164068:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.164080:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23989, 23989] 00000008:00010000:3.0:1775764896.164085:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.164088:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.164096:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164105:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98258944 98263039) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164123:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164132:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164138:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.164146:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98258944, 98263040) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.164151:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23989 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.164167:0:18302:0:(rw26.c:858:ll_write_end()) pos 98258944, len 4096, copied 4096 00000080:00200000:3.0:1775764896.164171:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000058ad29df[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.164174:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009b1f913c 17ffffc0001001 3:0 ffff90c21281be00 23989 lru 00000080:00200000:3.0:1775764896.164181:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a67b0b2c 23989: 1< 0 - - > 2< 98258944 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2355200 - | + - + - > 5< + - + - | 0 - | 3509 - -> 00000080:00200000:3.0:1775764896.164183:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000058ad29df 00000080:00200000:3.0:1775764896.164184:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.164186:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.164193:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.164200:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.164207:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.164215:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.164225:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.164228:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.164230:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.164240:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.164243:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.164246:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.164249:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164257:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.164265:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.164267:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.164271:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.164274:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23989, 23989] 00000080:00200000:3.0:1775764896.164284:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.164287:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.164317:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98263040 i_blocks 90112 00000080:00200000:3.0:1775764896.164321:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.164337:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98263040, count: 4096 00000080:00200000:3.0:1775764896.164348:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98263040, bytes: 4096 00000080:00200000:3.0:1775764896.164350:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.164354:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23991, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.164356:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98263040, 98267136) 0 0 00000080:00200000:3.0:1775764896.164358:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.164363:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23990, 23990] 00020000:00200000:3.0:1775764896.164366:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98263040 chunk: [98263040, 98267136) 98267136, 4096 00020000:00200000:3.0:1775764896.164369:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.164380:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23990, 23990] 00000008:00010000:3.0:1775764896.164385:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.164388:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.164396:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164405:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98263040 98267135) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164413:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164421:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164427:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.164435:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98263040, 98267136) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.164527:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23990 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.164544:0:18302:0:(rw26.c:858:ll_write_end()) pos 98263040, len 4096, copied 4096 00000080:00200000:3.0:1775764896.164549:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d07fab58[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.164551:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000ce19b5f 17ffffc0001001 3:0 ffff90c21281b540 23990 lru 00000080:00200000:3.0:1775764896.164557:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000dc252f0f 23990: 1< 0 - - > 2< 98263040 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2351104 - | + - + - > 5< + - + - | 0 - | 3510 - -> 00000080:00200000:3.0:1775764896.164559:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d07fab58 00000080:00200000:3.0:1775764896.164560:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.164562:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.164570:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.164578:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.164585:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.164593:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.164606:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.164608:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.164611:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.164621:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.164624:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.164627:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.164630:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164638:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.164648:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.164650:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.164654:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.164657:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23990, 23990] 00000080:00200000:3.0:1775764896.164668:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.164672:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.164703:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98267136 i_blocks 90112 00000080:00200000:3.0:1775764896.164706:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.164722:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98267136, count: 4096 00000080:00200000:3.0:1775764896.164733:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98267136, bytes: 4096 00000080:00200000:3.0:1775764896.164736:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.164740:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23992, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.164742:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98267136, 98271232) 0 0 00000080:00200000:3.0:1775764896.164745:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.164751:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23991, 23991] 00020000:00200000:3.0:1775764896.164754:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98267136 chunk: [98267136, 98271232) 98271232, 4096 00020000:00200000:3.0:1775764896.164757:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.164769:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23991, 23991] 00000008:00010000:3.0:1775764896.164774:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.164776:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.164784:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164792:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98267136 98271231) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164801:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164814:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164821:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.164829:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98267136, 98271232) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.164834:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23991 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.164851:0:18302:0:(rw26.c:858:ll_write_end()) pos 98267136, len 4096, copied 4096 00000080:00200000:3.0:1775764896.164856:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000a01588f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.164858:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000087843e39 17ffffc0001001 3:0 ffff90c21281b460 23991 lru 00000080:00200000:3.0:1775764896.164864:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006f747815 23991: 1< 0 - - > 2< 98267136 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2347008 - | + - + - > 5< + - + - | 0 - | 3511 - -> 00000080:00200000:3.0:1775764896.164866:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000a01588f 00000080:00200000:3.0:1775764896.164867:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.164892:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.164902:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.164910:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.164916:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.164924:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.164941:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.164944:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.164946:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.164956:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.164958:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.164961:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.164965:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.164973:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.164982:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.164984:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.164988:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.164991:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23991, 23991] 00000080:00200000:3.0:1775764896.165001:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.165004:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.165034:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98271232 i_blocks 90112 00000080:00200000:3.0:1775764896.165037:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.165053:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98271232, count: 4096 00000080:00200000:3.0:1775764896.165064:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98271232, bytes: 4096 00000080:00200000:3.0:1775764896.165066:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.165070:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23993, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.165072:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98271232, 98275328) 0 0 00000080:00200000:3.0:1775764896.165074:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.165079:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23992, 23992] 00020000:00200000:3.0:1775764896.165083:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98271232 chunk: [98271232, 98275328) 98275328, 4096 00020000:00200000:3.0:1775764896.165085:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.165097:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23992, 23992] 00000008:00010000:3.0:1775764896.165102:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.165105:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.165112:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165136:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98271232 98275327) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165144:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165151:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165157:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.165164:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98271232, 98275328) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.165169:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23992 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.165186:0:18302:0:(rw26.c:858:ll_write_end()) pos 98271232, len 4096, copied 4096 00000080:00200000:3.0:1775764896.165190:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006e1b1524[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.165193:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000042c5ca78 17ffffc0001001 3:0 ffff90c21281ba80 23992 lru 00000080:00200000:3.0:1775764896.165199:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e8f3798a 23992: 1< 0 - - > 2< 98271232 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2342912 - | + - + - > 5< + - + - | 0 - | 3512 - -> 00000080:00200000:3.0:1775764896.165201:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006e1b1524 00000080:00200000:3.0:1775764896.165202:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.165204:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.165211:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.165219:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.165226:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.165234:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.165244:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.165246:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.165248:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.165257:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.165260:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.165262:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.165266:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165274:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.165282:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.165284:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.165288:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.165291:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23992, 23992] 00000080:00200000:3.0:1775764896.165301:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.165304:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.165331:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98275328 i_blocks 90112 00000080:00200000:3.0:1775764896.165335:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.165349:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98275328, count: 4096 00000080:00200000:3.0:1775764896.165359:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98275328, bytes: 4096 00000080:00200000:3.0:1775764896.165362:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.165366:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23994, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.165367:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98275328, 98279424) 0 0 00000080:00200000:3.0:1775764896.165370:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.165375:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23993, 23993] 00020000:00200000:3.0:1775764896.165378:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98275328 chunk: [98275328, 98279424) 98279424, 4096 00020000:00200000:3.0:1775764896.165381:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.165392:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23993, 23993] 00000008:00010000:3.0:1775764896.165397:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.165400:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.165407:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165416:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98275328 98279423) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165424:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165432:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165518:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.165529:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98275328, 98279424) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.165534:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23993 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.165550:0:18302:0:(rw26.c:858:ll_write_end()) pos 98275328, len 4096, copied 4096 00000080:00200000:3.0:1775764896.165554:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005ee3bd34[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.165558:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000007d30001 17ffffc0001001 3:0 ffff90c21281b700 23993 lru 00000080:00200000:3.0:1775764896.165564:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000062557102 23993: 1< 0 - - > 2< 98275328 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2338816 - | + - + - > 5< + - + - | 0 - | 3513 - -> 00000080:00200000:3.0:1775764896.165566:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005ee3bd34 00000080:00200000:3.0:1775764896.165567:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.165569:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.165576:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.165583:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.165589:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.165595:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.165604:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.165607:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.165609:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.165618:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.165620:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.165622:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.165626:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165633:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.165640:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.165642:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.165646:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.165648:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23993, 23993] 00000080:00200000:3.0:1775764896.165658:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.165661:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.165686:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98279424 i_blocks 90112 00000080:00200000:3.0:1775764896.165689:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.165704:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98279424, count: 4096 00000080:00200000:3.0:1775764896.165714:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98279424, bytes: 4096 00000080:00200000:3.0:1775764896.165717:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.165720:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23995, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.165722:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98279424, 98283520) 0 0 00000080:00200000:3.0:1775764896.165724:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.165729:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23994, 23994] 00020000:00200000:3.0:1775764896.165732:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98279424 chunk: [98279424, 98283520) 98283520, 4096 00020000:00200000:3.0:1775764896.165734:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.165744:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23994, 23994] 00000008:00010000:3.0:1775764896.165749:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.165752:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.165758:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165767:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98279424 98283519) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165775:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165782:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165789:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.165797:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98279424, 98283520) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.165802:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23994 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.165819:0:18302:0:(rw26.c:858:ll_write_end()) pos 98279424, len 4096, copied 4096 00000080:00200000:3.0:1775764896.165823:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000eaad2b20[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.165825:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000021ba0198 17ffffc0001001 3:0 ffff90c21281bc40 23994 lru 00000080:00200000:3.0:1775764896.165830:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000068369044 23994: 1< 0 - - > 2< 98279424 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2334720 - | + - + - > 5< + - + - | 0 - | 3514 - -> 00000080:00200000:3.0:1775764896.165833:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000eaad2b20 00000080:00200000:3.0:1775764896.165834:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.165836:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.165844:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.165851:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.165858:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.165894:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.165906:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.165909:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.165912:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.165921:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.165923:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.165926:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.165930:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.165938:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.165945:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.165947:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.165951:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.165954:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23994, 23994] 00000080:00200000:3.0:1775764896.165964:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.165968:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.165998:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98283520 i_blocks 90112 00000080:00200000:3.0:1775764896.166002:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.166018:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98283520, count: 4096 00000080:00200000:3.0:1775764896.166028:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98283520, bytes: 4096 00000080:00200000:3.0:1775764896.166031:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.166035:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23996, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.166036:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98283520, 98287616) 0 0 00000080:00200000:3.0:1775764896.166039:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.166044:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23995, 23995] 00020000:00200000:3.0:1775764896.166047:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98283520 chunk: [98283520, 98287616) 98287616, 4096 00020000:00200000:3.0:1775764896.166050:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.166061:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23995, 23995] 00000008:00010000:3.0:1775764896.166066:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.166068:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.166076:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166084:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98283520 98287615) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166091:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166097:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166104:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.166111:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98283520, 98287616) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.166129:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23995 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.166149:0:18302:0:(rw26.c:858:ll_write_end()) pos 98283520, len 4096, copied 4096 00000080:00200000:3.0:1775764896.166153:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007d5f9d37[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.166156:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000aca6a7ab 17ffffc0001001 3:0 ffff90c21281b8c0 23995 lru 00000080:00200000:3.0:1775764896.166162:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007d2dd0e5 23995: 1< 0 - - > 2< 98283520 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2330624 - | + - + - > 5< + - + - | 0 - | 3515 - -> 00000080:00200000:3.0:1775764896.166164:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007d5f9d37 00000080:00200000:3.0:1775764896.166165:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.166167:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.166175:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.166183:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.166190:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.166197:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.166208:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.166211:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.166213:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.166222:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.166225:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.166227:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.166232:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166239:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.166247:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.166250:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.166254:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.166256:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23995, 23995] 00000080:00200000:3.0:1775764896.166266:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.166269:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.166299:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98287616 i_blocks 90112 00000080:00200000:3.0:1775764896.166303:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.166319:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98287616, count: 4096 00000080:00200000:3.0:1775764896.166330:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98287616, bytes: 4096 00000080:00200000:3.0:1775764896.166333:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.166336:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23997, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.166338:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98287616, 98291712) 0 0 00000080:00200000:3.0:1775764896.166340:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.166345:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23996, 23996] 00020000:00200000:3.0:1775764896.166348:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98287616 chunk: [98287616, 98291712) 98291712, 4096 00020000:00200000:3.0:1775764896.166350:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.166361:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23996, 23996] 00000008:00010000:3.0:1775764896.166366:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.166369:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.166377:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166386:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98287616 98291711) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166395:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166402:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166414:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.166421:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98287616, 98291712) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.166426:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23996 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.166535:0:18302:0:(rw26.c:858:ll_write_end()) pos 98287616, len 4096, copied 4096 00000080:00200000:3.0:1775764896.166540:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000015407207[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.166543:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000029f374cd 17ffffc0001021 2:0 ffff90c21281bb60 23996 lru 00000080:00200000:3.0:1775764896.166548:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000033e4cef5 23996: 1< 0 - - > 2< 98287616 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2326528 - | + - + - > 5< + - + - | 0 - | 3516 - -> 00000080:00200000:3.0:1775764896.166550:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000015407207 00000080:00200000:3.0:1775764896.166551:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.166553:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.166561:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.166568:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.166575:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.166582:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.166594:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.166597:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.166600:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.166610:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.166613:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.166615:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.166619:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.166628:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.166834:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.166837:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.166842:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.166846:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23996, 23996] 00000080:00200000:3.0:1775764896.166856:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.166860:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.166925:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98291712 i_blocks 90112 00000080:00200000:3.0:1775764896.166930:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.166947:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98291712, count: 4096 00000080:00200000:3.0:1775764896.166958:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98291712, bytes: 4096 00000080:00200000:3.0:1775764896.166961:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.166965:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23998, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.166966:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98291712, 98295808) 0 0 00000080:00200000:3.0:1775764896.166969:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.166975:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23997, 23997] 00020000:00200000:3.0:1775764896.166978:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98291712 chunk: [98291712, 98295808) 98295808, 4096 00020000:00200000:3.0:1775764896.166980:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.166992:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23997, 23997] 00000008:00010000:3.0:1775764896.166999:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.167001:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.167010:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167019:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98291712 98295807) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167027:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167035:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167042:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.167051:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98291712, 98295808) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.167057:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23997 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.167074:0:18302:0:(rw26.c:858:ll_write_end()) pos 98291712, len 4096, copied 4096 00000080:00200000:3.0:1775764896.167079:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ba61571c[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.167082:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003e9cc3f7 17ffffc0001001 3:0 ffff90c21281b000 23997 lru 00000080:00200000:3.0:1775764896.167087:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000fb27ec2 23997: 1< 0 - - > 2< 98291712 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2322432 - | + - + - > 5< + - + - | 0 - | 3517 - -> 00000080:00200000:3.0:1775764896.167089:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ba61571c 00000080:00200000:3.0:1775764896.167090:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.167093:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.167102:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.167110:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.167129:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.167137:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.167149:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.167152:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.167155:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.167165:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.167167:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.167170:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.167175:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167183:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.167191:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.167194:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.167198:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.167201:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23997, 23997] 00000080:00200000:3.0:1775764896.167212:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.167215:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.167245:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98295808 i_blocks 90112 00000080:00200000:3.0:1775764896.167249:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.167265:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98295808, count: 4096 00000080:00200000:3.0:1775764896.167277:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98295808, bytes: 4096 00000080:00200000:3.0:1775764896.167279:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.167284:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 23999, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.167285:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98295808, 98299904) 0 0 00000080:00200000:3.0:1775764896.167288:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.167293:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23998, 23998] 00020000:00200000:3.0:1775764896.167297:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98295808 chunk: [98295808, 98299904) 98299904, 4096 00020000:00200000:3.0:1775764896.167299:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.167311:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23998, 23998] 00000008:00010000:3.0:1775764896.167316:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.167318:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.167326:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167336:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98295808 98299903) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167344:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167351:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167358:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.167365:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98295808, 98299904) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.167371:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23998 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.167388:0:18302:0:(rw26.c:858:ll_write_end()) pos 98295808, len 4096, copied 4096 00000080:00200000:3.0:1775764896.167393:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e1f648d3[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.167396:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ff3a475a 17ffffc0001001 3:0 ffff90c21281bee0 23998 lru 00000080:00200000:3.0:1775764896.167402:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e00d1a39 23998: 1< 0 - - > 2< 98295808 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2318336 - | + - + - > 5< + - + - | 0 - | 3518 - -> 00000080:00200000:3.0:1775764896.167404:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e1f648d3 00000080:00200000:3.0:1775764896.167405:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.167407:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.167415:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.167423:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.167430:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.167572:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.167588:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.167591:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.167593:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.167604:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.167607:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.167609:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.167614:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167622:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.167631:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.167634:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.167639:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.167642:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23998, 23998] 00000080:00200000:3.0:1775764896.167653:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.167657:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.167690:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98299904 i_blocks 90112 00000080:00200000:3.0:1775764896.167693:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.167709:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98299904, count: 4096 00000080:00200000:3.0:1775764896.167720:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98299904, bytes: 4096 00000080:00200000:3.0:1775764896.167723:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.167727:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24000, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.167728:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98299904, 98304000) 0 0 00000080:00200000:3.0:1775764896.167731:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.167736:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [23999, 23999] 00020000:00200000:3.0:1775764896.167739:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98299904 chunk: [98299904, 98304000) 98304000, 4096 00020000:00200000:3.0:1775764896.167742:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.167754:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [23999, 23999] 00000008:00010000:3.0:1775764896.167759:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.167762:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.167770:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167779:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98299904 98303999) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167788:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167801:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167808:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.167816:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98299904, 98304000) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.167821:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 23999 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.167851:0:18302:0:(rw26.c:858:ll_write_end()) pos 98299904, len 4096, copied 4096 00000080:00200000:3.0:1775764896.167856:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d0b4c114[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.167859:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000081129b32 17ffffc0001001 3:0 ffff90c21281c2a0 23999 lru 00000080:00200000:3.0:1775764896.167865:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000076de983 23999: 1< 0 - - > 2< 98299904 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2314240 - | + - + - > 5< + - + - | 0 - | 3519 - -> 00000080:00200000:3.0:1775764896.167867:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d0b4c114 00000080:00200000:3.0:1775764896.167889:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.167893:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.167902:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.167910:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.167916:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.167922:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.167937:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.167940:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.167942:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.167952:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.167955:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.167957:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.167961:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.167969:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.167977:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.167979:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.167984:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.167987:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [23999, 23999] 00000080:00200000:3.0:1775764896.167997:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.168000:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.168028:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98304000 i_blocks 90112 00000080:00200000:3.0:1775764896.168031:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.168047:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98304000, count: 4096 00000080:00200000:3.0:1775764896.168058:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98304000, bytes: 4096 00000080:00200000:3.0:1775764896.168061:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.168065:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24001, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.168067:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98304000, 98308096) 0 0 00000080:00200000:3.0:1775764896.168069:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.168075:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24000, 24000] 00020000:00200000:3.0:1775764896.168078:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98304000 chunk: [98304000, 98308096) 98308096, 4096 00020000:00200000:3.0:1775764896.168081:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.168092:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24000, 24000] 00000008:00010000:3.0:1775764896.168098:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.168101:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.168108:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168128:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98304000 98308095) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168137:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168144:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168150:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.168157:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98304000, 98308096) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.168162:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24000 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.168182:0:18302:0:(rw26.c:858:ll_write_end()) pos 98304000, len 4096, copied 4096 00000080:00200000:3.0:1775764896.168186:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c33130c0[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.168189:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000010feb2e0 17ffffc0001001 3:0 ffff90c21281cd20 24000 lru 00000080:00200000:3.0:1775764896.168194:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c00632e7 24000: 1< 0 - - > 2< 98304000 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2310144 - | + - + - > 5< + - + - | 0 - | 3520 - -> 00000080:00200000:3.0:1775764896.168196:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c33130c0 00000080:00200000:3.0:1775764896.168197:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.168199:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.168207:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.168214:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.168220:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.168226:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.168237:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.168240:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.168242:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.168251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.168254:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.168257:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.168261:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168268:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.168275:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.168277:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.168281:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.168284:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24000, 24000] 00000080:00200000:3.0:1775764896.168296:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.168299:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.168329:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98308096 i_blocks 90112 00000080:00200000:3.0:1775764896.168333:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.168349:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98308096, count: 4096 00000080:00200000:3.0:1775764896.168359:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98308096, bytes: 4096 00000080:00200000:3.0:1775764896.168361:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.168365:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24002, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.168367:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98308096, 98312192) 0 0 00000080:00200000:3.0:1775764896.168369:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.168374:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24001, 24001] 00020000:00200000:3.0:1775764896.168377:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98308096 chunk: [98308096, 98312192) 98312192, 4096 00020000:00200000:3.0:1775764896.168379:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.168390:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24001, 24001] 00000008:00010000:3.0:1775764896.168395:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.168398:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.168405:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168414:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98308096 98312191) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168423:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168430:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168515:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.168527:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98308096, 98312192) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.168533:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24001 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.168552:0:18302:0:(rw26.c:858:ll_write_end()) pos 98308096, len 4096, copied 4096 00000080:00200000:3.0:1775764896.168556:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000033e1e3ac[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.168559:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000820cf2b2 17ffffc0001001 3:0 ffff90c21281c380 24001 lru 00000080:00200000:3.0:1775764896.168565:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000049967049 24001: 1< 0 - - > 2< 98308096 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2306048 - | + - + - > 5< + - + - | 0 - | 3521 - -> 00000080:00200000:3.0:1775764896.168567:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000033e1e3ac 00000080:00200000:3.0:1775764896.168568:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.168570:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.168577:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.168585:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.168592:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.168599:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.168610:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.168613:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.168615:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.168624:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.168627:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.168630:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.168634:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168641:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.168649:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.168651:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.168655:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.168658:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24001, 24001] 00000080:00200000:3.0:1775764896.168669:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.168672:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.168701:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98312192 i_blocks 90112 00000080:00200000:3.0:1775764896.168704:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.168720:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98312192, count: 4096 00000080:00200000:3.0:1775764896.168730:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98312192, bytes: 4096 00000080:00200000:3.0:1775764896.168733:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.168737:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24003, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.168739:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98312192, 98316288) 0 0 00000080:00200000:3.0:1775764896.168742:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.168747:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24002, 24002] 00020000:00200000:3.0:1775764896.168750:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98312192 chunk: [98312192, 98316288) 98316288, 4096 00020000:00200000:3.0:1775764896.168753:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.168764:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24002, 24002] 00000008:00010000:3.0:1775764896.168770:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.168772:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.168781:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168790:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98312192 98316287) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168800:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168808:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168815:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.168823:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98312192, 98316288) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.168828:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24002 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.168845:0:18302:0:(rw26.c:858:ll_write_end()) pos 98312192, len 4096, copied 4096 00000080:00200000:3.0:1775764896.168849:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000397897df[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.168852:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f5683876 17ffffc0001001 3:0 ffff90c21281c620 24002 lru 00000080:00200000:3.0:1775764896.168858:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000091b67791 24002: 1< 0 - - > 2< 98312192 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2301952 - | + - + - > 5< + - + - | 0 - | 3522 - -> 00000080:00200000:3.0:1775764896.168860:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000397897df 00000080:00200000:3.0:1775764896.168861:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.168863:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.168895:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.168904:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.168911:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.168924:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.168935:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.168939:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.168941:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.168951:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.168953:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.168956:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.168960:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.168968:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.168976:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.168978:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.168982:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.168985:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24002, 24002] 00000080:00200000:3.0:1775764896.168995:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.168998:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.169027:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98316288 i_blocks 90112 00000080:00200000:3.0:1775764896.169031:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.169047:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98316288, count: 4096 00000080:00200000:3.0:1775764896.169057:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98316288, bytes: 4096 00000080:00200000:3.0:1775764896.169060:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.169063:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24004, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.169065:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98316288, 98320384) 0 0 00000080:00200000:3.0:1775764896.169067:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.169073:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24003, 24003] 00020000:00200000:3.0:1775764896.169076:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98316288 chunk: [98316288, 98320384) 98320384, 4096 00020000:00200000:3.0:1775764896.169078:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.169091:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24003, 24003] 00000008:00010000:3.0:1775764896.169096:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.169098:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.169106:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169126:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98316288 98320383) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169135:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169142:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169150:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.169158:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98316288, 98320384) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.169164:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24003 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.169181:0:18302:0:(rw26.c:858:ll_write_end()) pos 98316288, len 4096, copied 4096 00000080:00200000:3.0:1775764896.169186:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000031bffcbb[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.169189:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000fc6b125 17ffffc0001001 3:0 ffff90c21281ce00 24003 lru 00000080:00200000:3.0:1775764896.169195:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ca897765 24003: 1< 0 - - > 2< 98316288 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2297856 - | + - + - > 5< + - + - | 0 - | 3523 - -> 00000080:00200000:3.0:1775764896.169198:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000031bffcbb 00000080:00200000:3.0:1775764896.169199:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.169201:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.169209:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.169218:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.169225:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.169233:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.169245:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.169248:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.169250:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.169260:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.169263:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.169266:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.169270:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169279:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.169287:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.169290:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.169294:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.169297:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24003, 24003] 00000080:00200000:3.0:1775764896.169308:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.169311:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.169343:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98320384 i_blocks 90112 00000080:00200000:3.0:1775764896.169346:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.169362:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98320384, count: 4096 00000080:00200000:3.0:1775764896.169373:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98320384, bytes: 4096 00000080:00200000:3.0:1775764896.169376:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.169379:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24005, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.169381:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98320384, 98324480) 0 0 00000080:00200000:3.0:1775764896.169383:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.169388:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24004, 24004] 00020000:00200000:3.0:1775764896.169391:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98320384 chunk: [98320384, 98324480) 98324480, 4096 00020000:00200000:3.0:1775764896.169394:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.169405:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24004, 24004] 00000008:00010000:3.0:1775764896.169410:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.169413:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.169421:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169430:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98320384 98324479) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169520:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169529:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169542:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.169550:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98320384, 98324480) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.169555:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24004 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.169573:0:18302:0:(rw26.c:858:ll_write_end()) pos 98320384, len 4096, copied 4096 00000080:00200000:3.0:1775764896.169577:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000178af2b1[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.169580:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008e85bd51 17ffffc0001001 3:0 ffff90c21281c540 24004 lru 00000080:00200000:3.0:1775764896.169585:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000019f1285d 24004: 1< 0 - - > 2< 98320384 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2293760 - | + - + - > 5< + - + - | 0 - | 3524 - -> 00000080:00200000:3.0:1775764896.169587:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000178af2b1 00000080:00200000:3.0:1775764896.169589:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.169591:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.169599:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.169607:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.169614:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.169622:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.169634:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.169637:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.169639:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.169650:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.169652:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.169655:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.169659:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169667:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.169675:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.169678:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.169681:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.169684:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24004, 24004] 00000080:00200000:3.0:1775764896.169695:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.169698:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.169727:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98324480 i_blocks 90112 00000080:00200000:3.0:1775764896.169731:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.169747:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98324480, count: 4096 00000080:00200000:3.0:1775764896.169757:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98324480, bytes: 4096 00000080:00200000:3.0:1775764896.169760:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.169763:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24006, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.169765:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98324480, 98328576) 0 0 00000080:00200000:3.0:1775764896.169767:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.169772:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24005, 24005] 00020000:00200000:3.0:1775764896.169775:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98324480 chunk: [98324480, 98328576) 98328576, 4096 00020000:00200000:3.0:1775764896.169777:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.169787:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24005, 24005] 00000008:00010000:3.0:1775764896.169793:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.169796:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.169804:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169812:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98324480 98328575) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169821:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169828:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169835:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.169843:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98324480, 98328576) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.169847:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24005 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.169863:0:18302:0:(rw26.c:858:ll_write_end()) pos 98324480, len 4096, copied 4096 00000080:00200000:3.0:1775764896.169868:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000036b23743[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.169894:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000029a4b96c 17ffffc0001001 3:0 ffff90c21281c460 24005 lru 00000080:00200000:3.0:1775764896.169900:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000043fefea5 24005: 1< 0 - - > 2< 98324480 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2289664 - | + - + - > 5< + - + - | 0 - | 3525 - -> 00000080:00200000:3.0:1775764896.169902:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000036b23743 00000080:00200000:3.0:1775764896.169904:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.169906:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.169914:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.169922:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.169929:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.169937:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.169948:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.169951:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.169954:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.169963:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.169966:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.169969:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.169972:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.169979:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.169987:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.169990:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.169994:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.169997:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24005, 24005] 00000080:00200000:3.0:1775764896.170007:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.170010:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.170041:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98328576 i_blocks 90112 00000080:00200000:3.0:1775764896.170045:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.170061:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98328576, count: 4096 00000080:00200000:3.0:1775764896.170071:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98328576, bytes: 4096 00000080:00200000:3.0:1775764896.170074:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.170078:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24007, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.170079:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98328576, 98332672) 0 0 00000080:00200000:3.0:1775764896.170082:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.170087:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24006, 24006] 00020000:00200000:3.0:1775764896.170090:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98328576 chunk: [98328576, 98332672) 98332672, 4096 00020000:00200000:3.0:1775764896.170092:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.170103:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24006, 24006] 00000008:00010000:3.0:1775764896.170108:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.170110:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.170130:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170139:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98328576 98332671) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170147:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170154:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170161:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.170169:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98328576, 98332672) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.170175:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24006 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.170191:0:18302:0:(rw26.c:858:ll_write_end()) pos 98328576, len 4096, copied 4096 00000080:00200000:3.0:1775764896.170194:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cfcf7d36[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.170197:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003bf6f672 17ffffc0001001 3:0 ffff90c21281ca80 24006 lru 00000080:00200000:3.0:1775764896.170202:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000025fc5a3b 24006: 1< 0 - - > 2< 98328576 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2285568 - | + - + - > 5< + - + - | 0 - | 3526 - -> 00000080:00200000:3.0:1775764896.170205:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cfcf7d36 00000080:00200000:3.0:1775764896.170205:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.170207:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.170215:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.170223:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.170231:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.170238:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.170249:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.170251:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.170253:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.170263:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.170266:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.170268:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.170272:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170279:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.170287:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.170290:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.170293:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.170296:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24006, 24006] 00000080:00200000:3.0:1775764896.170307:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.170310:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.170340:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98332672 i_blocks 90112 00000080:00200000:3.0:1775764896.170344:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.170360:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98332672, count: 4096 00000080:00200000:3.0:1775764896.170370:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98332672, bytes: 4096 00000080:00200000:3.0:1775764896.170372:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.170376:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24008, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.170377:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98332672, 98336768) 0 0 00000080:00200000:3.0:1775764896.170380:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.170385:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24007, 24007] 00020000:00200000:3.0:1775764896.170388:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98332672 chunk: [98332672, 98336768) 98336768, 4096 00020000:00200000:3.0:1775764896.170390:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.170401:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24007, 24007] 00000008:00010000:3.0:1775764896.170407:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.170409:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.170418:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170426:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98332672 98336767) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170435:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170534:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170542:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.170551:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98332672, 98336768) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.170557:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24007 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.170576:0:18302:0:(rw26.c:858:ll_write_end()) pos 98332672, len 4096, copied 4096 00000080:00200000:3.0:1775764896.170580:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001ec626be[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.170583:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000dc123c39 17ffffc0001001 3:0 ffff90c21281c700 24007 lru 00000080:00200000:3.0:1775764896.170589:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001085cb08 24007: 1< 0 - - > 2< 98332672 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2281472 - | + - + - > 5< + - + - | 0 - | 3527 - -> 00000080:00200000:3.0:1775764896.170591:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001ec626be 00000080:00200000:3.0:1775764896.170592:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.170594:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.170602:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.170611:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.170617:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.170624:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.170640:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.170644:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.170646:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.170655:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.170658:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.170661:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.170665:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170673:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.170681:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.170684:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.170688:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.170691:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24007, 24007] 00000080:00200000:3.0:1775764896.170701:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.170705:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.170735:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98336768 i_blocks 90112 00000080:00200000:3.0:1775764896.170739:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.170756:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98336768, count: 4096 00000080:00200000:3.0:1775764896.170767:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98336768, bytes: 4096 00000080:00200000:3.0:1775764896.170770:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.170774:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24009, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.170775:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98336768, 98340864) 0 0 00000080:00200000:3.0:1775764896.170777:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.170783:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24008, 24008] 00020000:00200000:3.0:1775764896.170787:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98336768 chunk: [98336768, 98340864) 98340864, 4096 00020000:00200000:3.0:1775764896.170789:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.170801:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24008, 24008] 00000008:00010000:3.0:1775764896.170807:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.170809:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.170817:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170827:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98336768 98340863) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170835:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170842:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170850:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.170857:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98336768, 98340864) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.170862:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24008 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.170902:0:18302:0:(rw26.c:858:ll_write_end()) pos 98336768, len 4096, copied 4096 00000080:00200000:3.0:1775764896.170907:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000453ddddf[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.170910:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fd0bf051 17ffffc0001001 3:0 ffff90c21281cc40 24008 lru 00000080:00200000:3.0:1775764896.170916:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000830006e9 24008: 1< 0 - - > 2< 98336768 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2277376 - | + - + - > 5< + - + - | 0 - | 3528 - -> 00000080:00200000:3.0:1775764896.170918:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000453ddddf 00000080:00200000:3.0:1775764896.170919:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.170922:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.170930:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.170938:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.170945:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.170953:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.170965:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.170968:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.170970:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.170980:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.170983:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.170985:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.170989:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.170997:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.171006:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.171008:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.171012:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.171015:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24008, 24008] 00000080:00200000:3.0:1775764896.171026:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.171029:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.171059:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98340864 i_blocks 90112 00000080:00200000:3.0:1775764896.171063:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.171079:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98340864, count: 4096 00000080:00200000:3.0:1775764896.171089:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98340864, bytes: 4096 00000080:00200000:3.0:1775764896.171092:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.171095:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24010, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.171097:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98340864, 98344960) 0 0 00000080:00200000:3.0:1775764896.171100:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.171105:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24009, 24009] 00020000:00200000:3.0:1775764896.171108:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98340864 chunk: [98340864, 98344960) 98344960, 4096 00020000:00200000:3.0:1775764896.171110:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.171135:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24009, 24009] 00000008:00010000:3.0:1775764896.171141:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.171144:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.171152:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171161:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98340864 98344959) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171170:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171177:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171183:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.171192:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98340864, 98344960) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.171197:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24009 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.171213:0:18302:0:(rw26.c:858:ll_write_end()) pos 98340864, len 4096, copied 4096 00000080:00200000:3.0:1775764896.171218:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a74eec4e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.171221:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000dbbcb41b 17ffffc0001001 3:0 ffff90c21281c8c0 24009 lru 00000080:00200000:3.0:1775764896.171227:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007fee56f4 24009: 1< 0 - - > 2< 98340864 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2273280 - | + - + - > 5< + - + - | 0 - | 3529 - -> 00000080:00200000:3.0:1775764896.171229:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a74eec4e 00000080:00200000:3.0:1775764896.171230:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.171233:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.171241:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.171249:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.171255:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.171263:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.171274:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.171277:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.171280:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.171289:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.171291:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.171293:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.171298:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171306:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.171314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.171316:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.171321:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.171323:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24009, 24009] 00000080:00200000:3.0:1775764896.171334:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.171337:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.171366:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98344960 i_blocks 90112 00000080:00200000:3.0:1775764896.171369:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.171385:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98344960, count: 4096 00000080:00200000:3.0:1775764896.171395:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98344960, bytes: 4096 00000080:00200000:3.0:1775764896.171398:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.171402:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24011, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.171403:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98344960, 98349056) 0 0 00000080:00200000:3.0:1775764896.171406:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.171411:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24010, 24010] 00020000:00200000:3.0:1775764896.171414:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98344960 chunk: [98344960, 98349056) 98349056, 4096 00020000:00200000:3.0:1775764896.171416:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.171428:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24010, 24010] 00000008:00010000:3.0:1775764896.171433:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.171436:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.171524:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171534:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98344960 98349055) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171543:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171550:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171557:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.171564:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98344960, 98349056) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.171569:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24010 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.171586:0:18302:0:(rw26.c:858:ll_write_end()) pos 98344960, len 4096, copied 4096 00000080:00200000:3.0:1775764896.171590:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008d1b046f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.171593:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a518968d 17ffffc0001001 3:0 ffff90c21281cb60 24010 lru 00000080:00200000:3.0:1775764896.171598:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000097129bfe 24010: 1< 0 - - > 2< 98344960 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2269184 - | + - + - > 5< + - + - | 0 - | 3530 - -> 00000080:00200000:3.0:1775764896.171600:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008d1b046f 00000080:00200000:3.0:1775764896.171601:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.171603:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.171611:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.171618:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.171625:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.171637:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.171648:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.171651:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.171653:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.171663:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.171666:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.171668:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.171672:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171680:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.171687:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.171689:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.171693:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.171695:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24010, 24010] 00000080:00200000:3.0:1775764896.171706:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.171709:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.171735:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98349056 i_blocks 90112 00000080:00200000:3.0:1775764896.171739:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.171753:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98349056, count: 4096 00000080:00200000:3.0:1775764896.171762:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98349056, bytes: 4096 00000080:00200000:3.0:1775764896.171764:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.171768:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24012, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.171769:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98349056, 98353152) 0 0 00000080:00200000:3.0:1775764896.171772:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.171776:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24011, 24011] 00020000:00200000:3.0:1775764896.171779:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98349056 chunk: [98349056, 98353152) 98353152, 4096 00020000:00200000:3.0:1775764896.171781:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.171791:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24011, 24011] 00000008:00010000:3.0:1775764896.171795:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.171797:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.171804:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171812:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98349056 98353151) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171819:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171825:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171831:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.171837:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98349056, 98353152) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.171841:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24011 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.171863:0:18302:0:(rw26.c:858:ll_write_end()) pos 98349056, len 4096, copied 4096 00000080:00200000:3.0:1775764896.171866:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000093cdc99[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.171889:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f7c97d09 17ffffc0001021 2:0 ffff90c21281c000 24011 lru 00000080:00200000:3.0:1775764896.171895:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d5111b70 24011: 1< 0 - - > 2< 98349056 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2265088 - | + - + - > 5< + - + - | 0 - | 3531 - -> 00000080:00200000:3.0:1775764896.171897:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000093cdc99 00000080:00200000:3.0:1775764896.171899:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.171901:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.171909:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.171916:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.171923:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.171930:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.171940:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.171943:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.171945:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.171954:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.171957:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.171959:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.171963:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.171971:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.171979:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.171982:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.171986:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.171989:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24011, 24011] 00000080:00200000:3.0:1775764896.171999:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.172003:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.172030:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98353152 i_blocks 90112 00000080:00200000:3.0:1775764896.172035:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.172050:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98353152, count: 4096 00000080:00200000:3.0:1775764896.172060:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98353152, bytes: 4096 00000080:00200000:3.0:1775764896.172063:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.172067:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24013, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.172068:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98353152, 98357248) 0 0 00000080:00200000:3.0:1775764896.172070:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.172075:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24012, 24012] 00020000:00200000:3.0:1775764896.172078:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98353152 chunk: [98353152, 98357248) 98357248, 4096 00020000:00200000:3.0:1775764896.172080:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.172092:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24012, 24012] 00000008:00010000:3.0:1775764896.172097:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.172099:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.172107:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172127:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98353152 98357247) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172136:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172143:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172155:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.172163:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98353152, 98357248) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.172168:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24012 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.172185:0:18302:0:(rw26.c:858:ll_write_end()) pos 98353152, len 4096, copied 4096 00000080:00200000:3.0:1775764896.172189:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000031836867[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.172192:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006dc4b528 17ffffc0001001 3:0 ffff90c21281cee0 24012 lru 00000080:00200000:3.0:1775764896.172198:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000e45ce97 24012: 1< 0 - - > 2< 98353152 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2260992 - | + - + - > 5< + - + - | 0 - | 3532 - -> 00000080:00200000:3.0:1775764896.172201:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000031836867 00000080:00200000:3.0:1775764896.172202:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.172204:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.172212:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.172220:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.172228:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.172235:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.172247:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.172250:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.172252:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.172262:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.172264:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.172267:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.172270:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172277:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.172286:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.172288:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.172292:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.172295:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24012, 24012] 00000080:00200000:3.0:1775764896.172306:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.172309:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.172339:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98357248 i_blocks 90112 00000080:00200000:3.0:1775764896.172342:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.172358:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98357248, count: 4096 00000080:00200000:3.0:1775764896.172367:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98357248, bytes: 4096 00000080:00200000:3.0:1775764896.172370:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.172373:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24014, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.172375:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98357248, 98361344) 0 0 00000080:00200000:3.0:1775764896.172377:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.172382:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24013, 24013] 00020000:00200000:3.0:1775764896.172385:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98357248 chunk: [98357248, 98361344) 98361344, 4096 00020000:00200000:3.0:1775764896.172387:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.172398:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24013, 24013] 00000008:00010000:3.0:1775764896.172404:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.172406:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.172414:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172423:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98357248 98361343) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172431:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172521:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172530:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.172539:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98357248, 98361344) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.172544:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24013 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.172562:0:18302:0:(rw26.c:858:ll_write_end()) pos 98357248, len 4096, copied 4096 00000080:00200000:3.0:1775764896.172566:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000040a10d53[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.172569:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ff883a14 17ffffc0001001 3:0 ffff90c21281c0e0 24013 lru 00000080:00200000:3.0:1775764896.172575:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a577fcae 24013: 1< 0 - - > 2< 98357248 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2256896 - | + - + - > 5< + - + - | 0 - | 3533 - -> 00000080:00200000:3.0:1775764896.172577:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000040a10d53 00000080:00200000:3.0:1775764896.172578:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.172581:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.172589:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.172596:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.172602:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.172610:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.172621:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.172624:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.172626:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.172637:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.172640:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.172642:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.172646:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172653:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.172662:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.172664:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.172668:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.172671:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24013, 24013] 00000080:00200000:3.0:1775764896.172682:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.172685:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.172715:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98361344 i_blocks 90112 00000080:00200000:3.0:1775764896.172718:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.172732:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98361344, count: 4096 00000080:00200000:3.0:1775764896.172742:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98361344, bytes: 4096 00000080:00200000:3.0:1775764896.172744:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.172747:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24015, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.172749:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98361344, 98365440) 0 0 00000080:00200000:3.0:1775764896.172751:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.172756:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24014, 24014] 00020000:00200000:3.0:1775764896.172758:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98361344 chunk: [98361344, 98365440) 98365440, 4096 00020000:00200000:3.0:1775764896.172760:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.172770:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24014, 24014] 00000008:00010000:3.0:1775764896.172775:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.172778:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.172785:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172794:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98361344 98365439) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172802:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172809:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172815:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.172822:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98361344, 98365440) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.172828:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24014 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.172844:0:18302:0:(rw26.c:858:ll_write_end()) pos 98361344, len 4096, copied 4096 00000080:00200000:3.0:1775764896.172848:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000087f7bddd[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.172851:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000016f81db8 17ffffc0001001 3:0 ffff90c21281c1c0 24014 lru 00000080:00200000:3.0:1775764896.172857:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007cdbe2c5 24014: 1< 0 - - > 2< 98361344 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2252800 - | + - + - > 5< + - + - | 0 - | 3534 - -> 00000080:00200000:3.0:1775764896.172859:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000087f7bddd 00000080:00200000:3.0:1775764896.172861:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.172863:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.172895:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.172904:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.172910:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.172918:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.172929:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.172932:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.172935:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.172944:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.172946:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.172949:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.172953:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.172961:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.172969:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.172971:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.172975:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.172978:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24014, 24014] 00000080:00200000:3.0:1775764896.172988:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.172991:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.173021:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98365440 i_blocks 90112 00000080:00200000:3.0:1775764896.173025:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.173041:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98365440, count: 4096 00000080:00200000:3.0:1775764896.173051:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98365440, bytes: 4096 00000080:00200000:3.0:1775764896.173054:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.173057:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24016, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.173059:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98365440, 98369536) 0 0 00000080:00200000:3.0:1775764896.173061:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.173066:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24015, 24015] 00020000:00200000:3.0:1775764896.173069:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98365440 chunk: [98365440, 98369536) 98369536, 4096 00020000:00200000:3.0:1775764896.173071:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.173082:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24015, 24015] 00000008:00010000:3.0:1775764896.173087:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.173090:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.173097:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173105:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98365440 98369535) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173112:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173138:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173145:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.173153:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98365440, 98369536) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.173157:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24015 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.173172:0:18302:0:(rw26.c:858:ll_write_end()) pos 98365440, len 4096, copied 4096 00000080:00200000:3.0:1775764896.173176:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c4878ea5[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.173179:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f1866100 17ffffc0001001 3:0 ffff90c21281c7e0 24015 lru 00000080:00200000:3.0:1775764896.173183:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e5266b49 24015: 1< 0 - - > 2< 98365440 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2248704 - | + - + - > 5< + - + - | 0 - | 3535 - -> 00000080:00200000:3.0:1775764896.173185:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c4878ea5 00000080:00200000:3.0:1775764896.173186:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.173188:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.173195:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.173203:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.173210:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.173218:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.173234:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.173236:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.173239:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.173248:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.173250:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.173252:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.173256:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173264:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.173272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.173274:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.173278:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.173280:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24015, 24015] 00000080:00200000:3.0:1775764896.173292:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.173639:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.173677:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98369536 i_blocks 90112 00000080:00200000:3.0:1775764896.173682:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.173699:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98369536, count: 4096 00000080:00200000:3.0:1775764896.173709:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98369536, bytes: 4096 00000080:00200000:3.0:1775764896.173712:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.173716:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24017, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.173717:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98369536, 98373632) 0 0 00000080:00200000:3.0:1775764896.173720:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.173726:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24016, 24016] 00020000:00200000:3.0:1775764896.173729:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98369536 chunk: [98369536, 98373632) 98373632, 4096 00020000:00200000:3.0:1775764896.173732:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.173744:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24016, 24016] 00000008:00010000:3.0:1775764896.173750:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.173752:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.173761:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173769:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98369536 98373631) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173777:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173785:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173791:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.173798:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98369536, 98373632) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.173804:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24016 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.173823:0:18302:0:(rw26.c:858:ll_write_end()) pos 98369536, len 4096, copied 4096 00000080:00200000:3.0:1775764896.173827:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e37a0738[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.173830:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000241dfa89 17ffffc0001001 3:0 ffff90c21281c9a0 24016 lru 00000080:00200000:3.0:1775764896.173836:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000905e440 24016: 1< 0 - - > 2< 98369536 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2244608 - | + - + - > 5< + - + - | 0 - | 3536 - -> 00000080:00200000:3.0:1775764896.173838:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e37a0738 00000080:00200000:3.0:1775764896.173839:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.173841:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.173851:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.173860:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.173867:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.173899:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.173912:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.173915:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.173918:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.173929:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.173932:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.173934:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.173938:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.173946:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.173955:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.173957:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.173961:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.173964:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24016, 24016] 00000080:00200000:3.0:1775764896.173976:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.173979:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.174009:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98373632 i_blocks 90112 00000080:00200000:3.0:1775764896.174013:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.174029:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98373632, count: 4096 00000080:00200000:3.0:1775764896.174040:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98373632, bytes: 4096 00000080:00200000:3.0:1775764896.174043:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.174046:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24018, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.174048:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98373632, 98377728) 0 0 00000080:00200000:3.0:1775764896.174050:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.174056:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24017, 24017] 00020000:00200000:3.0:1775764896.174059:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98373632 chunk: [98373632, 98377728) 98377728, 4096 00020000:00200000:3.0:1775764896.174061:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.174073:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24017, 24017] 00000008:00010000:3.0:1775764896.174078:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.174081:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.174089:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174098:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98373632 98377727) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174107:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174146:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174154:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.174163:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98373632, 98377728) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.174168:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24017 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.174197:0:18302:0:(rw26.c:858:ll_write_end()) pos 98373632, len 4096, copied 4096 00000080:00200000:3.0:1775764896.174202:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ebb6c6f8[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.174204:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c4f3e1c4 17ffffc0001001 3:0 ffff90c21281db60 24017 lru 00000080:00200000:3.0:1775764896.174210:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e2f66b58 24017: 1< 0 - - > 2< 98373632 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2240512 - | + - + - > 5< + - + - | 0 - | 3537 - -> 00000080:00200000:3.0:1775764896.174212:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ebb6c6f8 00000080:00200000:3.0:1775764896.174213:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.174216:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.174224:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.174232:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.174239:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.174247:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.174258:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.174261:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.174263:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.174273:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.174276:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.174279:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.174283:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174291:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.174299:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.174302:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.174307:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.174310:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24017, 24017] 00000080:00200000:3.0:1775764896.174322:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.174325:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.174354:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98377728 i_blocks 90112 00000080:00200000:3.0:1775764896.174357:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.174373:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98377728, count: 4096 00000080:00200000:3.0:1775764896.174383:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98377728, bytes: 4096 00000080:00200000:3.0:1775764896.174386:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.174390:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24019, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.174391:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98377728, 98381824) 0 0 00000080:00200000:3.0:1775764896.174394:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.174399:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24018, 24018] 00020000:00200000:3.0:1775764896.174402:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98377728 chunk: [98377728, 98381824) 98381824, 4096 00020000:00200000:3.0:1775764896.174405:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.174416:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24018, 24018] 00000008:00010000:3.0:1775764896.174422:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.174424:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.174432:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174529:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98377728 98381823) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174540:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174547:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174554:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.174564:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98377728, 98381824) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.174571:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24018 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.174589:0:18302:0:(rw26.c:858:ll_write_end()) pos 98377728, len 4096, copied 4096 00000080:00200000:3.0:1775764896.174594:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fe6d3063[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.174597:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f799912d 17ffffc0001001 3:0 ffff90c21281d000 24018 lru 00000080:00200000:3.0:1775764896.174603:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006fe3a852 24018: 1< 0 - - > 2< 98377728 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2236416 - | + - + - > 5< + - + - | 0 - | 3538 - -> 00000080:00200000:3.0:1775764896.174605:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fe6d3063 00000080:00200000:3.0:1775764896.174606:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.174609:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.174618:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.174626:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.174633:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.174646:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.174658:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.174661:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.174663:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.174675:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.174678:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.174681:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.174685:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174693:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.174701:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.174704:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.174708:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.174711:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24018, 24018] 00000080:00200000:3.0:1775764896.174722:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.174726:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.174757:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98381824 i_blocks 90112 00000080:00200000:3.0:1775764896.174760:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.174776:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98381824, count: 4096 00000080:00200000:3.0:1775764896.174787:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98381824, bytes: 4096 00000080:00200000:3.0:1775764896.174790:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.174794:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24020, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.174795:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98381824, 98385920) 0 0 00000080:00200000:3.0:1775764896.174798:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.174804:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24019, 24019] 00020000:00200000:3.0:1775764896.174806:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98381824 chunk: [98381824, 98385920) 98385920, 4096 00020000:00200000:3.0:1775764896.174809:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.174821:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24019, 24019] 00000008:00010000:3.0:1775764896.174826:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.174829:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.174836:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174844:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98381824 98385919) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174851:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174858:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.174865:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.174896:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98381824, 98385920) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.174901:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24019 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.174920:0:18302:0:(rw26.c:858:ll_write_end()) pos 98381824, len 4096, copied 4096 00000080:00200000:3.0:1775764896.174924:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e53f4501[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.174927:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f1200634 17ffffc0001001 3:0 ffff90c21281dee0 24019 lru 00000080:00200000:3.0:1775764896.174932:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000037f4466e 24019: 1< 0 - - > 2< 98381824 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2232320 - | + - + - > 5< + - + - | 0 - | 3539 - -> 00000080:00200000:3.0:1775764896.174935:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e53f4501 00000080:00200000:3.0:1775764896.174936:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.174939:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.174946:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.174954:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.174961:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.174968:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.174979:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.174982:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.174984:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.174993:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.174996:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.174998:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.175002:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175010:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.175018:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.175021:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.175025:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.175028:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24019, 24019] 00000080:00200000:3.0:1775764896.175038:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.175041:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.175071:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98385920 i_blocks 90112 00000080:00200000:3.0:1775764896.175074:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.175089:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98385920, count: 4096 00000080:00200000:3.0:1775764896.175098:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98385920, bytes: 4096 00000080:00200000:3.0:1775764896.175101:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.175104:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24021, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.175108:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98385920, 98390016) 0 0 00000080:00200000:3.0:1775764896.175110:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.175127:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24020, 24020] 00020000:00200000:3.0:1775764896.175130:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98385920 chunk: [98385920, 98390016) 98390016, 4096 00020000:00200000:3.0:1775764896.175132:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.175144:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24020, 24020] 00000008:00010000:3.0:1775764896.175149:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.175152:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.175159:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175168:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98385920 98390015) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175176:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175184:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175197:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.175205:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98385920, 98390016) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.175210:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24020 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.175228:0:18302:0:(rw26.c:858:ll_write_end()) pos 98385920, len 4096, copied 4096 00000080:00200000:3.0:1775764896.175232:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000287f38c6[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.175235:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000084c57f18 17ffffc0001001 3:0 ffff90c21281d0e0 24020 lru 00000080:00200000:3.0:1775764896.175241:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003c52031b 24020: 1< 0 - - > 2< 98385920 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2228224 - | + - + - > 5< + - + - | 0 - | 3540 - -> 00000080:00200000:3.0:1775764896.175243:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000287f38c6 00000080:00200000:3.0:1775764896.175244:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.175246:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.175254:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.175262:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.175269:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.175277:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.175288:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.175292:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.175294:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.175304:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.175307:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.175310:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.175314:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175321:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.175330:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.175332:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.175337:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.175339:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24020, 24020] 00000080:00200000:3.0:1775764896.175350:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.175353:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.175382:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98390016 i_blocks 90112 00000080:00200000:3.0:1775764896.175386:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.175402:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98390016, count: 4096 00000080:00200000:3.0:1775764896.175412:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98390016, bytes: 4096 00000080:00200000:3.0:1775764896.175415:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.175419:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24022, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.175420:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98390016, 98394112) 0 0 00000080:00200000:3.0:1775764896.175422:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.175428:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24021, 24021] 00020000:00200000:3.0:1775764896.175431:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98390016 chunk: [98390016, 98394112) 98394112, 4096 00020000:00200000:3.0:1775764896.175433:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.175540:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24021, 24021] 00000008:00010000:3.0:1775764896.175546:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.175549:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.175558:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175567:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98390016 98394111) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175576:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175583:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175591:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.175600:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98390016, 98394112) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.175605:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24021 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.175621:0:18302:0:(rw26.c:858:ll_write_end()) pos 98390016, len 4096, copied 4096 00000080:00200000:3.0:1775764896.175626:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000022cf804e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.175629:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f6e4df91 17ffffc0001001 3:0 ffff90c21281d1c0 24021 lru 00000080:00200000:3.0:1775764896.175634:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000de9fb743 24021: 1< 0 - - > 2< 98390016 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2224128 - | + - + - > 5< + - + - | 0 - | 3541 - -> 00000080:00200000:3.0:1775764896.175637:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000022cf804e 00000080:00200000:3.0:1775764896.175638:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.175640:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.175649:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.175657:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.175664:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.175672:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.175684:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.175687:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.175690:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.175700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.175703:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.175705:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.175709:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175717:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.175726:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.175728:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.175732:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.175735:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24021, 24021] 00000080:00200000:3.0:1775764896.175746:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.175749:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.175774:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98394112 i_blocks 90112 00000080:00200000:3.0:1775764896.175778:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.175793:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98394112, count: 4096 00000080:00200000:3.0:1775764896.175802:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98394112, bytes: 4096 00000080:00200000:3.0:1775764896.175805:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.175809:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24023, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.175810:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98394112, 98398208) 0 0 00000080:00200000:3.0:1775764896.175813:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.175818:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24022, 24022] 00020000:00200000:3.0:1775764896.175821:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98394112 chunk: [98394112, 98398208) 98398208, 4096 00020000:00200000:3.0:1775764896.175824:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.175833:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24022, 24022] 00000008:00010000:3.0:1775764896.175838:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.175841:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.175848:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175857:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98394112 98398207) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175865:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175896:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.175903:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.175911:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98394112, 98398208) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.175917:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24022 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.175935:0:18302:0:(rw26.c:858:ll_write_end()) pos 98394112, len 4096, copied 4096 00000080:00200000:3.0:1775764896.175939:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d1e7b92c[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.175942:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ce8a5ccf 17ffffc0001001 3:0 ffff90c21281d7e0 24022 lru 00000080:00200000:3.0:1775764896.175948:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000088780536 24022: 1< 0 - - > 2< 98394112 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2220032 - | + - + - > 5< + - + - | 0 - | 3542 - -> 00000080:00200000:3.0:1775764896.175950:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d1e7b92c 00000080:00200000:3.0:1775764896.175951:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.175953:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.175962:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.175968:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.175975:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.175982:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.175992:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.175995:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.175997:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.176007:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.176009:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.176011:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.176015:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176021:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.176030:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.176032:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.176036:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.176039:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24022, 24022] 00000080:00200000:3.0:1775764896.176049:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.176052:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.176083:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98398208 i_blocks 90112 00000080:00200000:3.0:1775764896.176087:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.176102:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98398208, count: 4096 00000080:00200000:3.0:1775764896.176112:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98398208, bytes: 4096 00000080:00200000:3.0:1775764896.176126:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.176129:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24024, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.176131:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98398208, 98402304) 0 0 00000080:00200000:3.0:1775764896.176133:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.176138:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24023, 24023] 00020000:00200000:3.0:1775764896.176141:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98398208 chunk: [98398208, 98402304) 98402304, 4096 00020000:00200000:3.0:1775764896.176143:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.176154:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24023, 24023] 00000008:00010000:3.0:1775764896.176159:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.176161:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.176168:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176176:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98398208 98402303) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176183:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176195:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176202:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.176209:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98398208, 98402304) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.176214:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24023 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.176230:0:18302:0:(rw26.c:858:ll_write_end()) pos 98398208, len 4096, copied 4096 00000080:00200000:3.0:1775764896.176234:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008d757356[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.176237:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000019052c50 17ffffc0001001 3:0 ffff90c21281d9a0 24023 lru 00000080:00200000:3.0:1775764896.176243:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000035255925 24023: 1< 0 - - > 2< 98398208 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2215936 - | + - + - > 5< + - + - | 0 - | 3543 - -> 00000080:00200000:3.0:1775764896.176245:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008d757356 00000080:00200000:3.0:1775764896.176246:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.176248:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.176255:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.176263:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.176268:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.176276:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.176294:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.176297:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.176299:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.176310:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.176313:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.176316:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.176320:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176328:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.176337:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.176339:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.176343:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.176346:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24023, 24023] 00000080:00200000:3.0:1775764896.176356:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.176359:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.176388:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98402304 i_blocks 90112 00000080:00200000:3.0:1775764896.176392:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.176407:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98402304, count: 4096 00000080:00200000:3.0:1775764896.176417:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98402304, bytes: 4096 00000080:00200000:3.0:1775764896.176420:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.176424:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24025, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.176426:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98402304, 98406400) 0 0 00000080:00200000:3.0:1775764896.176428:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.176433:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24024, 24024] 00020000:00200000:3.0:1775764896.176437:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98402304 chunk: [98402304, 98406400) 98406400, 4096 00020000:00200000:3.0:1775764896.176520:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.176535:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24024, 24024] 00000008:00010000:3.0:1775764896.176541:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.176543:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.176551:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176560:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98402304 98406399) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176568:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176576:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176582:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.176590:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98402304, 98406400) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.176594:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24024 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.176611:0:18302:0:(rw26.c:858:ll_write_end()) pos 98402304, len 4096, copied 4096 00000080:00200000:3.0:1775764896.176616:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000736ff369[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.176619:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006289c8de 17ffffc0001001 3:0 ffff90c21281d2a0 24024 lru 00000080:00200000:3.0:1775764896.176624:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c0e102d8 24024: 1< 0 - - > 2< 98402304 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2211840 - | + - + - > 5< + - + - | 0 - | 3544 - -> 00000080:00200000:3.0:1775764896.176626:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000736ff369 00000080:00200000:3.0:1775764896.176627:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.176629:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.176637:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.176645:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.176652:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.176659:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.176670:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.176673:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.176675:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.176685:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.176688:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.176690:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.176694:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176702:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.176710:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.176713:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.176717:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.176719:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24024, 24024] 00000080:00200000:3.0:1775764896.176731:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.176734:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.176764:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98406400 i_blocks 90112 00000080:00200000:3.0:1775764896.176768:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.176783:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98406400, count: 4096 00000080:00200000:3.0:1775764896.176793:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98406400, bytes: 4096 00000080:00200000:3.0:1775764896.176796:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.176799:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24026, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.176801:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98406400, 98410496) 0 0 00000080:00200000:3.0:1775764896.176803:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.176808:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24025, 24025] 00020000:00200000:3.0:1775764896.176811:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98406400 chunk: [98406400, 98410496) 98410496, 4096 00020000:00200000:3.0:1775764896.176814:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.176825:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24025, 24025] 00000008:00010000:3.0:1775764896.176830:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.176832:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.176840:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176849:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98406400 98410495) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176857:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176863:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.176890:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.176900:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98406400, 98410496) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.176905:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24025 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.176921:0:18302:0:(rw26.c:858:ll_write_end()) pos 98406400, len 4096, copied 4096 00000080:00200000:3.0:1775764896.176925:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d92f3a76[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.176928:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cec90044 17ffffc0001001 3:0 ffff90c21281dd20 24025 lru 00000080:00200000:3.0:1775764896.176933:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000022524923 24025: 1< 0 - - > 2< 98406400 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2207744 - | + - + - > 5< + - + - | 0 - | 3545 - -> 00000080:00200000:3.0:1775764896.176935:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d92f3a76 00000080:00200000:3.0:1775764896.176937:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.176939:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.176947:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.176955:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.176962:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.176969:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.176980:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.176983:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.176985:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.176995:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.176998:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.177000:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.177004:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177011:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.177020:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.177022:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.177026:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.177029:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24025, 24025] 00000080:00200000:3.0:1775764896.177040:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.177043:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.177073:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98410496 i_blocks 90112 00000080:00200000:3.0:1775764896.177076:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.177093:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98410496, count: 4096 00000080:00200000:3.0:1775764896.177103:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98410496, bytes: 4096 00000080:00200000:3.0:1775764896.177106:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.177110:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24027, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.177111:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98410496, 98414592) 0 0 00000080:00200000:3.0:1775764896.177126:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.177132:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24026, 24026] 00020000:00200000:3.0:1775764896.177135:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98410496 chunk: [98410496, 98414592) 98414592, 4096 00020000:00200000:3.0:1775764896.177138:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.177150:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24026, 24026] 00000008:00010000:3.0:1775764896.177155:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.177158:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.177164:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177173:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98410496 98414591) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177183:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177190:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177197:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.177205:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98410496, 98414592) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.177210:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24026 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.177234:0:18302:0:(rw26.c:858:ll_write_end()) pos 98410496, len 4096, copied 4096 00000080:00200000:3.0:1775764896.177238:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001857f4ee[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.177241:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000bd497fb 17ffffc0001021 2:0 ffff90c21281d380 24026 lru 00000080:00200000:3.0:1775764896.177247:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002dba562f 24026: 1< 0 - - > 2< 98410496 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2203648 - | + - + - > 5< + - + - | 0 - | 3546 - -> 00000080:00200000:3.0:1775764896.177249:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001857f4ee 00000080:00200000:3.0:1775764896.177250:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.177252:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.177260:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.177269:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.177275:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.177288:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.177300:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.177303:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.177305:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.177315:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.177317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.177319:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.177323:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177331:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.177340:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.177342:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.177346:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.177349:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24026, 24026] 00000080:00200000:3.0:1775764896.177359:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.177362:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.177392:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98414592 i_blocks 90112 00000080:00200000:3.0:1775764896.177395:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.177412:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98414592, count: 4096 00000080:00200000:3.0:1775764896.177423:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98414592, bytes: 4096 00000080:00200000:3.0:1775764896.177425:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.177430:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24028, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.177431:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98414592, 98418688) 0 0 00000080:00200000:3.0:1775764896.177434:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.177527:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24027, 24027] 00020000:00200000:3.0:1775764896.177531:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98414592 chunk: [98414592, 98418688) 98418688, 4096 00020000:00200000:3.0:1775764896.177534:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.177547:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24027, 24027] 00000008:00010000:3.0:1775764896.177553:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.177556:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.177564:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177574:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98414592 98418687) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177583:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177592:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177600:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.177608:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98414592, 98418688) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.177614:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24027 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.177630:0:18302:0:(rw26.c:858:ll_write_end()) pos 98414592, len 4096, copied 4096 00000080:00200000:3.0:1775764896.177634:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008dba30a8[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.177637:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000e437e21 17ffffc0001001 3:0 ffff90c21281d620 24027 lru 00000080:00200000:3.0:1775764896.177643:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003d78f6bb 24027: 1< 0 - - > 2< 98414592 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2199552 - | + - + - > 5< + - + - | 0 - | 3547 - -> 00000080:00200000:3.0:1775764896.177645:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008dba30a8 00000080:00200000:3.0:1775764896.177646:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.177649:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.177657:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.177665:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.177672:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.177679:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.177690:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.177694:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.177696:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.177707:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.177710:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.177712:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.177717:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177725:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.177734:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.177737:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.177741:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.177744:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24027, 24027] 00000080:00200000:3.0:1775764896.177755:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.177759:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.177789:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98418688 i_blocks 90112 00000080:00200000:3.0:1775764896.177792:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.177807:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98418688, count: 4096 00000080:00200000:3.0:1775764896.177818:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98418688, bytes: 4096 00000080:00200000:3.0:1775764896.177821:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.177824:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24029, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.177826:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98418688, 98422784) 0 0 00000080:00200000:3.0:1775764896.177828:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.177834:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24028, 24028] 00020000:00200000:3.0:1775764896.177837:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98418688 chunk: [98418688, 98422784) 98422784, 4096 00020000:00200000:3.0:1775764896.177839:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.177851:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24028, 24028] 00000008:00010000:3.0:1775764896.177856:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.177859:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.177867:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177901:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98418688 98422783) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177909:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177917:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.177929:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.177936:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98418688, 98422784) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.177941:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24028 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.177959:0:18302:0:(rw26.c:858:ll_write_end()) pos 98418688, len 4096, copied 4096 00000080:00200000:3.0:1775764896.177964:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a057f737[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.177966:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001b1574bc 17ffffc0001001 3:0 ffff90c21281de00 24028 lru 00000080:00200000:3.0:1775764896.177972:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000078a14d11 24028: 1< 0 - - > 2< 98418688 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2195456 - | + - + - > 5< + - + - | 0 - | 3548 - -> 00000080:00200000:3.0:1775764896.177973:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a057f737 00000080:00200000:3.0:1775764896.177974:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.177977:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.177984:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.177992:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.177999:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178007:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.178019:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.178022:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.178025:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.178034:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.178037:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.178039:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.178043:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178051:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.178059:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.178061:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.178065:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.178068:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24028, 24028] 00000080:00200000:3.0:1775764896.178079:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.178082:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.178109:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98422784 i_blocks 90112 00000080:00200000:3.0:1775764896.178124:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.178140:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98422784, count: 4096 00000080:00200000:3.0:1775764896.178150:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98422784, bytes: 4096 00000080:00200000:3.0:1775764896.178152:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.178155:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24030, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.178157:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98422784, 98426880) 0 0 00000080:00200000:3.0:1775764896.178160:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.178164:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24029, 24029] 00020000:00200000:3.0:1775764896.178167:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98422784 chunk: [98422784, 98426880) 98426880, 4096 00020000:00200000:3.0:1775764896.178170:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.178180:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24029, 24029] 00000008:00010000:3.0:1775764896.178185:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.178187:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.178195:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178204:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98422784 98426879) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178212:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178219:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178226:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.178233:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98422784, 98426880) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.178238:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24029 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.178253:0:18302:0:(rw26.c:858:ll_write_end()) pos 98422784, len 4096, copied 4096 00000080:00200000:3.0:1775764896.178257:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000562e12d7[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.178259:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000012599bb3 17ffffc0001001 3:0 ffff90c21281d540 24029 lru 00000080:00200000:3.0:1775764896.178264:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007a4c37ed 24029: 1< 0 - - > 2< 98422784 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2191360 - | + - + - > 5< + - + - | 0 - | 3549 - -> 00000080:00200000:3.0:1775764896.178267:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000562e12d7 00000080:00200000:3.0:1775764896.178268:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.178270:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.178277:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178285:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178292:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178300:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.178310:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.178313:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.178315:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.178325:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.178327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.178329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.178333:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178341:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.178348:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.178350:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.178355:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.178357:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24029, 24029] 00000080:00200000:3.0:1775764896.178367:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.178371:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.178399:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98426880 i_blocks 90112 00000080:00200000:3.0:1775764896.178403:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.178419:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98426880, count: 4096 00000080:00200000:3.0:1775764896.178429:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98426880, bytes: 4096 00000080:00200000:3.0:1775764896.178431:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.178435:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24031, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.178436:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98426880, 98430976) 0 0 00000080:00200000:3.0:1775764896.178514:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.178521:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24030, 24030] 00020000:00200000:3.0:1775764896.178524:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98426880 chunk: [98426880, 98430976) 98430976, 4096 00020000:00200000:3.0:1775764896.178526:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.178539:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24030, 24030] 00000008:00010000:3.0:1775764896.178543:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.178545:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.178552:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178560:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98426880 98430975) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178568:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178574:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178580:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.178587:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98426880, 98430976) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.178591:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24030 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.178607:0:18302:0:(rw26.c:858:ll_write_end()) pos 98426880, len 4096, copied 4096 00000080:00200000:3.0:1775764896.178610:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005e26191f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.178613:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000862fbc32 17ffffc0001001 3:0 ffff90c21281d460 24030 lru 00000080:00200000:3.0:1775764896.178618:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000524232d0 24030: 1< 0 - - > 2< 98426880 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2187264 - | + - + - > 5< + - + - | 0 - | 3550 - -> 00000080:00200000:3.0:1775764896.178620:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005e26191f 00000080:00200000:3.0:1775764896.178621:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.178623:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.178631:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178639:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178645:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178652:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.178663:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.178665:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.178668:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.178677:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.178680:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.178682:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.178685:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178692:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.178701:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.178703:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.178706:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.178709:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24030, 24030] 00000080:00200000:3.0:1775764896.178720:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.178723:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.178752:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98430976 i_blocks 90112 00000080:00200000:3.0:1775764896.178756:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.178771:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98430976, count: 4096 00000080:00200000:3.0:1775764896.178782:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98430976, bytes: 4096 00000080:00200000:3.0:1775764896.178784:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.178787:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24032, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.178789:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98430976, 98435072) 0 0 00000080:00200000:3.0:1775764896.178791:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.178796:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24031, 24031] 00020000:00200000:3.0:1775764896.178799:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98430976 chunk: [98430976, 98435072) 98435072, 4096 00020000:00200000:3.0:1775764896.178801:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.178812:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24031, 24031] 00000008:00010000:3.0:1775764896.178817:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.178819:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.178827:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178836:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98430976 98435071) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178844:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178856:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.178863:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.178905:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98430976, 98435072) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.178911:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24031 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.178927:0:18302:0:(rw26.c:858:ll_write_end()) pos 98430976, len 4096, copied 4096 00000080:00200000:3.0:1775764896.178931:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ce313bb9[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.178934:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000985f78e1 17ffffc0001001 3:0 ffff90c21281da80 24031 lru 00000080:00200000:3.0:1775764896.178940:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f404e84d 24031: 1< 0 - - > 2< 98430976 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2183168 - | + - + - > 5< + - + - | 0 - | 3551 - -> 00000080:00200000:3.0:1775764896.178941:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ce313bb9 00000080:00200000:3.0:1775764896.178942:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.178945:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.178953:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178961:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178967:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.178974:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.178990:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.178993:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.178995:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.179005:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.179008:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.179011:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.179015:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179023:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.179032:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.179034:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.179038:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.179041:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24031, 24031] 00000080:00200000:3.0:1775764896.179052:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.179055:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.179086:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98435072 i_blocks 90112 00000080:00200000:3.0:1775764896.179089:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.179106:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98435072, count: 4096 00000080:00200000:3.0:1775764896.179127:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98435072, bytes: 4096 00000080:00200000:3.0:1775764896.179130:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.179133:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24033, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.179135:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98435072, 98439168) 0 0 00000080:00200000:3.0:1775764896.179137:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.179142:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24032, 24032] 00020000:00200000:3.0:1775764896.179145:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98435072 chunk: [98435072, 98439168) 98439168, 4096 00020000:00200000:3.0:1775764896.179147:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.179159:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24032, 24032] 00000008:00010000:3.0:1775764896.179163:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.179166:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.179174:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179182:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98435072 98439167) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179190:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179198:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179204:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.179213:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98435072, 98439168) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.179218:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24032 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.179234:0:18302:0:(rw26.c:858:ll_write_end()) pos 98435072, len 4096, copied 4096 00000080:00200000:3.0:1775764896.179238:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d7779de1[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.179241:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000435762c6 17ffffc0001001 3:0 ffff90c21281d700 24032 lru 00000080:00200000:3.0:1775764896.179246:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f388262a 24032: 1< 0 - - > 2< 98435072 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2179072 - | + - + - > 5< + - + - | 0 - | 3552 - -> 00000080:00200000:3.0:1775764896.179248:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d7779de1 00000080:00200000:3.0:1775764896.179249:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.179251:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.179260:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.179267:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.179273:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.179280:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.179291:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.179294:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.179296:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.179305:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.179308:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.179310:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.179314:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179321:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.179329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.179331:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.179335:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.179338:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24032, 24032] 00000080:00200000:3.0:1775764896.179349:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.179352:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.179381:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98439168 i_blocks 90112 00000080:00200000:3.0:1775764896.179385:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.179400:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98439168, count: 4096 00000080:00200000:3.0:1775764896.179411:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98439168, bytes: 4096 00000080:00200000:3.0:1775764896.179414:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.179417:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24034, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.179419:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98439168, 98443264) 0 0 00000080:00200000:3.0:1775764896.179421:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.179426:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24033, 24033] 00020000:00200000:3.0:1775764896.179429:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98439168 chunk: [98439168, 98443264) 98443264, 4096 00020000:00200000:3.0:1775764896.179431:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.179528:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24033, 24033] 00000008:00010000:3.0:1775764896.179534:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.179536:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.179544:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179552:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98439168 98443263) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179561:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179569:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179575:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.179584:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98439168, 98443264) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.179588:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24033 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.179602:0:18302:0:(rw26.c:858:ll_write_end()) pos 98439168, len 4096, copied 4096 00000080:00200000:3.0:1775764896.179606:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000557bc99[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.179609:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000067399138 17ffffc0001001 3:0 ffff90c21281dc40 24033 lru 00000080:00200000:3.0:1775764896.179615:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007e63b7d0 24033: 1< 0 - - > 2< 98439168 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2174976 - | + - + - > 5< + - + - | 0 - | 3553 - -> 00000080:00200000:3.0:1775764896.179617:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000557bc99 00000080:00200000:3.0:1775764896.179618:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.179620:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.179627:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.179634:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.179642:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.179649:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.179659:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.179661:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.179663:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.179673:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.179675:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.179677:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.179681:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179689:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.179697:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.179699:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.179703:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.179706:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24033, 24033] 00000080:00200000:3.0:1775764896.179716:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.179718:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.179743:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98443264 i_blocks 90112 00000080:00200000:3.0:1775764896.179747:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.179761:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98443264, count: 4096 00000080:00200000:3.0:1775764896.179770:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98443264, bytes: 4096 00000080:00200000:3.0:1775764896.179773:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.179777:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24035, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.179778:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98443264, 98447360) 0 0 00000080:00200000:3.0:1775764896.179781:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.179786:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24034, 24034] 00020000:00200000:3.0:1775764896.179788:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98443264 chunk: [98443264, 98447360) 98447360, 4096 00020000:00200000:3.0:1775764896.179791:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.179800:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24034, 24034] 00000008:00010000:3.0:1775764896.179804:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.179806:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.179813:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179822:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98443264 98447359) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179831:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179839:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179846:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.179853:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98443264, 98447360) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.179857:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24034 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.179893:0:18302:0:(rw26.c:858:ll_write_end()) pos 98443264, len 4096, copied 4096 00000080:00200000:3.0:1775764896.179896:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000eb80ceff[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.179899:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000059dd5f9c 17ffffc0001001 3:0 ffff90c21281d8c0 24034 lru 00000080:00200000:3.0:1775764896.179904:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000038940fbf 24034: 1< 0 - - > 2< 98443264 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2170880 - | + - + - > 5< + - + - | 0 - | 3554 - -> 00000080:00200000:3.0:1775764896.179906:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000eb80ceff 00000080:00200000:3.0:1775764896.179907:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.179909:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.179916:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.179923:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.179931:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.179944:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.179955:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.179958:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.179960:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.179970:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.179972:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.179975:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.179978:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.179986:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.179994:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.179996:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.180000:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.180003:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24034, 24034] 00000080:00200000:3.0:1775764896.180014:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.180017:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.180046:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98447360 i_blocks 90112 00000080:00200000:3.0:1775764896.180050:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.180064:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98447360, count: 4096 00000080:00200000:3.0:1775764896.180074:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98447360, bytes: 4096 00000080:00200000:3.0:1775764896.180076:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.180080:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24036, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.180082:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98447360, 98451456) 0 0 00000080:00200000:3.0:1775764896.180084:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.180089:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24035, 24035] 00020000:00200000:3.0:1775764896.180092:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98447360 chunk: [98447360, 98451456) 98451456, 4096 00020000:00200000:3.0:1775764896.180094:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.180104:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24035, 24035] 00000008:00010000:3.0:1775764896.180108:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.180111:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.180128:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180138:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98447360 98451455) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180146:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180154:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180160:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.180168:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98447360, 98451456) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.180173:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24035 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.180195:0:18302:0:(rw26.c:858:ll_write_end()) pos 98447360, len 4096, copied 4096 00000080:00200000:3.0:1775764896.180199:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000095c51db1[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.180202:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000820fe4b5 17ffffc0001001 3:0 ffff90c21281eee0 24035 lru 00000080:00200000:3.0:1775764896.180208:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a48882a1 24035: 1< 0 - - > 2< 98447360 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2166784 - | + - + - > 5< + - + - | 0 - | 3555 - -> 00000080:00200000:3.0:1775764896.180210:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000095c51db1 00000080:00200000:3.0:1775764896.180211:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.180214:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.180220:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.180228:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.180236:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.180243:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.180253:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.180255:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.180257:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.180265:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.180267:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.180270:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.180273:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180283:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.180291:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.180293:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.180297:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.180299:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24035, 24035] 00000080:00200000:3.0:1775764896.180309:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.180312:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.180337:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98451456 i_blocks 90112 00000080:00200000:3.0:1775764896.180341:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.180355:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98451456, count: 4096 00000080:00200000:3.0:1775764896.180364:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98451456, bytes: 4096 00000080:00200000:3.0:1775764896.180367:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.180370:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24037, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.180372:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98451456, 98455552) 0 0 00000080:00200000:3.0:1775764896.180374:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.180379:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24036, 24036] 00020000:00200000:3.0:1775764896.180381:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98451456 chunk: [98451456, 98455552) 98455552, 4096 00020000:00200000:3.0:1775764896.180384:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.180393:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24036, 24036] 00000008:00010000:3.0:1775764896.180398:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.180400:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.180407:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180416:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98451456 98455551) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180424:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180432:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180521:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.180529:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98451456, 98455552) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.180535:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24036 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.180549:0:18302:0:(rw26.c:858:ll_write_end()) pos 98451456, len 4096, copied 4096 00000080:00200000:3.0:1775764896.180552:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e3f409a8[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.180555:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c8a6f922 17ffffc0001001 3:0 ffff90c21281e0e0 24036 lru 00000080:00200000:3.0:1775764896.180561:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004ca082d1 24036: 1< 0 - - > 2< 98451456 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2162688 - | + - + - > 5< + - + - | 0 - | 3556 - -> 00000080:00200000:3.0:1775764896.180563:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e3f409a8 00000080:00200000:3.0:1775764896.180564:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.180566:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.180573:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.180580:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.180587:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.180594:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.180604:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.180607:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.180609:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.180618:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.180620:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.180623:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.180627:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180634:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.180642:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.180644:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.180648:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.180650:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24036, 24036] 00000080:00200000:3.0:1775764896.180660:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.180663:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.180688:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98455552 i_blocks 90112 00000080:00200000:3.0:1775764896.180691:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.180705:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98455552, count: 4096 00000080:00200000:3.0:1775764896.180714:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98455552, bytes: 4096 00000080:00200000:3.0:1775764896.180717:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.180720:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24038, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.180722:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98455552, 98459648) 0 0 00000080:00200000:3.0:1775764896.180724:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.180729:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24037, 24037] 00020000:00200000:3.0:1775764896.180731:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98455552 chunk: [98455552, 98459648) 98459648, 4096 00020000:00200000:3.0:1775764896.180734:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.180743:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24037, 24037] 00000008:00010000:3.0:1775764896.180747:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.180750:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.180757:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180766:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98455552 98459647) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180775:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180781:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180788:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.180795:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98455552, 98459648) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.180799:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24037 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.180814:0:18302:0:(rw26.c:858:ll_write_end()) pos 98455552, len 4096, copied 4096 00000080:00200000:3.0:1775764896.180818:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000766f10c[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.180821:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000072e184dc 17ffffc0001001 3:0 ffff90c21281e1c0 24037 lru 00000080:00200000:3.0:1775764896.180826:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000004833617 24037: 1< 0 - - > 2< 98455552 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2158592 - | + - + - > 5< + - + - | 0 - | 3557 - -> 00000080:00200000:3.0:1775764896.180829:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000766f10c 00000080:00200000:3.0:1775764896.180830:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.180832:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.180838:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.180845:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.180853:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.180861:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.180892:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.180895:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.180896:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.180904:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.180906:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.180909:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.180912:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.180920:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.180928:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.180930:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.180933:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.180936:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24037, 24037] 00000080:00200000:3.0:1775764896.180945:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.180948:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.180976:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98459648 i_blocks 90112 00000080:00200000:3.0:1775764896.180980:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.180995:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98459648, count: 4096 00000080:00200000:3.0:1775764896.181005:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98459648, bytes: 4096 00000080:00200000:3.0:1775764896.181007:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.181011:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24039, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.181012:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98459648, 98463744) 0 0 00000080:00200000:3.0:1775764896.181014:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.181019:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24038, 24038] 00020000:00200000:3.0:1775764896.181022:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98459648 chunk: [98459648, 98463744) 98463744, 4096 00020000:00200000:3.0:1775764896.181024:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.181034:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24038, 24038] 00000008:00010000:3.0:1775764896.181038:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.181040:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.181047:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181055:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98459648 98463743) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181064:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181072:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181080:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.181086:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98459648, 98463744) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.181090:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24038 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.181104:0:18302:0:(rw26.c:858:ll_write_end()) pos 98459648, len 4096, copied 4096 00000080:00200000:3.0:1775764896.181108:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000045d00944[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.181111:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003e84bd58 17ffffc0001001 3:0 ffff90c21281e7e0 24038 lru 00000080:00200000:3.0:1775764896.181127:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d414ff78 24038: 1< 0 - - > 2< 98459648 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2154496 - | + - + - > 5< + - + - | 0 - | 3558 - -> 00000080:00200000:3.0:1775764896.181129:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000045d00944 00000080:00200000:3.0:1775764896.181131:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.181133:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.181140:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.181148:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.181155:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.181162:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.181172:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.181174:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.181176:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.181184:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.181186:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.181188:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.181192:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181199:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.181208:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.181210:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.181214:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.181216:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24038, 24038] 00000080:00200000:3.0:1775764896.181226:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.181229:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.181253:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98463744 i_blocks 90112 00000080:00200000:3.0:1775764896.181256:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.181269:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98463744, count: 4096 00000080:00200000:3.0:1775764896.181278:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98463744, bytes: 4096 00000080:00200000:3.0:1775764896.181281:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.181284:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24040, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.181286:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98463744, 98467840) 0 0 00000080:00200000:3.0:1775764896.181288:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.181293:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24039, 24039] 00020000:00200000:3.0:1775764896.181296:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98463744 chunk: [98463744, 98467840) 98467840, 4096 00020000:00200000:3.0:1775764896.181298:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.181307:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24039, 24039] 00000008:00010000:3.0:1775764896.181311:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.181314:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.181321:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181330:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98463744 98467839) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181338:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181351:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181358:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.181365:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98463744, 98467840) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.181369:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24039 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.181386:0:18302:0:(rw26.c:858:ll_write_end()) pos 98463744, len 4096, copied 4096 00000080:00200000:3.0:1775764896.181389:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000088e6964f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.181392:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000067ae36d4 17ffffc0001001 3:0 ffff90c21281e9a0 24039 lru 00000080:00200000:3.0:1775764896.181397:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ed066576 24039: 1< 0 - - > 2< 98463744 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2150400 - | + - + - > 5< + - + - | 0 - | 3559 - -> 00000080:00200000:3.0:1775764896.181399:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000088e6964f 00000080:00200000:3.0:1775764896.181400:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.181401:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.181408:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.181416:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.181423:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.181430:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.181520:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.181523:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.181526:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.181535:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.181537:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.181540:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.181544:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181553:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.181561:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.181563:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.181566:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.181568:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24039, 24039] 00000080:00200000:3.0:1775764896.181579:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.181581:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.181606:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98467840 i_blocks 90112 00000080:00200000:3.0:1775764896.181609:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.181623:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98467840, count: 4096 00000080:00200000:3.0:1775764896.181632:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98467840, bytes: 4096 00000080:00200000:3.0:1775764896.181635:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.181638:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24041, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.181640:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98467840, 98471936) 0 0 00000080:00200000:3.0:1775764896.181642:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.181647:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24040, 24040] 00020000:00200000:3.0:1775764896.181650:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98467840 chunk: [98467840, 98471936) 98471936, 4096 00020000:00200000:3.0:1775764896.181653:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.181662:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24040, 24040] 00000008:00010000:3.0:1775764896.181666:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.181669:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.181676:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181685:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98467840 98471935) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181694:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181701:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181708:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.181715:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98467840, 98471936) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.181720:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24040 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.181735:0:18302:0:(rw26.c:858:ll_write_end()) pos 98467840, len 4096, copied 4096 00000080:00200000:3.0:1775764896.181739:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000037f3de4d[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.181742:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006224845f 17ffffc0001001 3:0 ffff90c21281e2a0 24040 lru 00000080:00200000:3.0:1775764896.181747:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004693639f 24040: 1< 0 - - > 2< 98467840 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2146304 - | + - + - > 5< + - + - | 0 - | 3560 - -> 00000080:00200000:3.0:1775764896.181749:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000037f3de4d 00000080:00200000:3.0:1775764896.181751:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.181752:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.181759:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.181767:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.181775:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.181782:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.181791:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.181794:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.181795:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.181803:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.181805:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.181807:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.181811:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181818:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.181826:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.181829:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.181833:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.181835:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24040, 24040] 00000080:00200000:3.0:1775764896.181846:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.181848:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.181893:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98471936 i_blocks 90112 00000080:00200000:3.0:1775764896.181896:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.181911:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98471936, count: 4096 00000080:00200000:3.0:1775764896.181920:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98471936, bytes: 4096 00000080:00200000:3.0:1775764896.181923:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.181926:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24042, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.181928:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98471936, 98476032) 0 0 00000080:00200000:3.0:1775764896.181930:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.181935:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24041, 24041] 00020000:00200000:3.0:1775764896.181938:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98471936 chunk: [98471936, 98476032) 98476032, 4096 00020000:00200000:3.0:1775764896.181940:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.181950:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24041, 24041] 00000008:00010000:3.0:1775764896.181954:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.181957:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.181965:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181974:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98471936 98476031) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181983:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181990:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.181997:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.182004:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98471936, 98476032) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.182008:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24041 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.182030:0:18302:0:(rw26.c:858:ll_write_end()) pos 98471936, len 4096, copied 4096 00000080:00200000:3.0:1775764896.182034:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006c2a2321[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.182037:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b66a3f1d 17ffffc0001021 2:0 ffff90c21281ed20 24041 lru 00000080:00200000:3.0:1775764896.182043:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e55a9576 24041: 1< 0 - - > 2< 98471936 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2142208 - | + - + - > 5< + - + - | 0 - | 3561 - -> 00000080:00200000:3.0:1775764896.182045:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006c2a2321 00000080:00200000:3.0:1775764896.182046:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.182048:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.182054:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182062:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182069:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182076:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.182086:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.182088:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.182090:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.182098:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.182100:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.182102:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.182107:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182127:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.182136:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.182139:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.182143:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.182145:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24041, 24041] 00000080:00200000:3.0:1775764896.182155:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.182158:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.182183:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98476032 i_blocks 90112 00000080:00200000:3.0:1775764896.182186:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.182200:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98476032, count: 4096 00000080:00200000:3.0:1775764896.182209:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98476032, bytes: 4096 00000080:00200000:3.0:1775764896.182211:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.182214:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24043, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.182216:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98476032, 98480128) 0 0 00000080:00200000:3.0:1775764896.182218:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.182223:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24042, 24042] 00020000:00200000:3.0:1775764896.182226:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98476032 chunk: [98476032, 98480128) 98480128, 4096 00020000:00200000:3.0:1775764896.182228:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.182237:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24042, 24042] 00000008:00010000:3.0:1775764896.182242:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.182244:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.182251:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182260:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98476032 98480127) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182269:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182277:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182284:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.182291:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98476032, 98480128) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.182294:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24042 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.182308:0:18302:0:(rw26.c:858:ll_write_end()) pos 98476032, len 4096, copied 4096 00000080:00200000:3.0:1775764896.182311:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000018240fc2[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.182314:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000058898b32 17ffffc0001001 3:0 ffff90c21281e380 24042 lru 00000080:00200000:3.0:1775764896.182319:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000084567f8f 24042: 1< 0 - - > 2< 98476032 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2138112 - | + - + - > 5< + - + - | 0 - | 3562 - -> 00000080:00200000:3.0:1775764896.182321:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000018240fc2 00000080:00200000:3.0:1775764896.182322:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.182324:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.182330:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182338:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182345:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182358:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.182368:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.182371:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.182373:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.182380:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.182382:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.182385:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.182389:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182397:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.182405:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.182407:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.182410:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.182412:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24042, 24042] 00000080:00200000:3.0:1775764896.182422:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.182425:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.182529:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98480128 i_blocks 90112 00000080:00200000:3.0:1775764896.182532:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.182548:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98480128, count: 4096 00000080:00200000:3.0:1775764896.182557:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98480128, bytes: 4096 00000080:00200000:3.0:1775764896.182559:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.182563:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24044, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.182564:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98480128, 98484224) 0 0 00000080:00200000:3.0:1775764896.182567:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.182572:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24043, 24043] 00020000:00200000:3.0:1775764896.182575:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98480128 chunk: [98480128, 98484224) 98484224, 4096 00020000:00200000:3.0:1775764896.182577:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.182587:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24043, 24043] 00000008:00010000:3.0:1775764896.182592:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.182594:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.182601:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182610:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98480128 98484223) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182618:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182625:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182631:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.182639:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98480128, 98484224) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.182643:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24043 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.182658:0:18302:0:(rw26.c:858:ll_write_end()) pos 98480128, len 4096, copied 4096 00000080:00200000:3.0:1775764896.182661:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cfdd8ac0[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.182664:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000b8db308 17ffffc0001001 3:0 ffff90c21281e620 24043 lru 00000080:00200000:3.0:1775764896.182670:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003a979713 24043: 1< 0 - - > 2< 98480128 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2134016 - | + - + - > 5< + - + - | 0 - | 3563 - -> 00000080:00200000:3.0:1775764896.182672:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cfdd8ac0 00000080:00200000:3.0:1775764896.182674:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.182676:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.182682:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182691:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182698:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182705:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.182715:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.182717:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.182719:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.182727:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.182729:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.182731:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.182735:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182743:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.182750:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.182753:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.182756:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.182759:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24043, 24043] 00000080:00200000:3.0:1775764896.182769:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.182771:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.182796:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98484224 i_blocks 90112 00000080:00200000:3.0:1775764896.182799:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.182812:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98484224, count: 4096 00000080:00200000:3.0:1775764896.182821:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98484224, bytes: 4096 00000080:00200000:3.0:1775764896.182824:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.182827:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24045, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.182828:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98484224, 98488320) 0 0 00000080:00200000:3.0:1775764896.182831:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.182835:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24044, 24044] 00020000:00200000:3.0:1775764896.182838:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98484224 chunk: [98484224, 98488320) 98488320, 4096 00020000:00200000:3.0:1775764896.182841:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.182850:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24044, 24044] 00000008:00010000:3.0:1775764896.182854:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.182856:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.182863:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182891:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98484224 98488319) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182900:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182907:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.182920:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.182927:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98484224, 98488320) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.182931:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24044 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.182945:0:18302:0:(rw26.c:858:ll_write_end()) pos 98484224, len 4096, copied 4096 00000080:00200000:3.0:1775764896.182949:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d6d39cb2[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.182951:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006fdfde16 17ffffc0001001 3:0 ffff90c21281ee00 24044 lru 00000080:00200000:3.0:1775764896.182956:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002c14a556 24044: 1< 0 - - > 2< 98484224 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2129920 - | + - + - > 5< + - + - | 0 - | 3564 - -> 00000080:00200000:3.0:1775764896.182958:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d6d39cb2 00000080:00200000:3.0:1775764896.182960:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.182962:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.182969:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182977:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182984:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.182991:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.183001:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.183004:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.183006:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.183015:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.183018:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.183020:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.183024:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183032:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.183040:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.183042:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.183045:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.183047:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24044, 24044] 00000080:00200000:3.0:1775764896.183057:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.183060:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.183084:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98488320 i_blocks 90112 00000080:00200000:3.0:1775764896.183087:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.183101:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98488320, count: 4096 00000080:00200000:3.0:1775764896.183110:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98488320, bytes: 4096 00000080:00200000:3.0:1775764896.183112:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.183128:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24046, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.183129:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98488320, 98492416) 0 0 00000080:00200000:3.0:1775764896.183131:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.183136:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24045, 24045] 00020000:00200000:3.0:1775764896.183139:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98488320 chunk: [98488320, 98492416) 98492416, 4096 00020000:00200000:3.0:1775764896.183141:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.183152:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24045, 24045] 00000008:00010000:3.0:1775764896.183156:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.183159:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.183167:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183175:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98488320 98492415) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183182:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183189:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183195:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.183203:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98488320, 98492416) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.183208:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24045 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.183223:0:18302:0:(rw26.c:858:ll_write_end()) pos 98488320, len 4096, copied 4096 00000080:00200000:3.0:1775764896.183227:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007c3bd24b[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.183229:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000058df01db 17ffffc0001001 3:0 ffff90c21281e540 24045 lru 00000080:00200000:3.0:1775764896.183235:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005a261087 24045: 1< 0 - - > 2< 98488320 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2125824 - | + - + - > 5< + - + - | 0 - | 3565 - -> 00000080:00200000:3.0:1775764896.183237:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007c3bd24b 00000080:00200000:3.0:1775764896.183238:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.183239:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.183247:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.183254:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.183261:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.183268:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.183280:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.183282:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.183284:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.183293:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.183296:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.183298:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.183302:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183310:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.183318:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.183320:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.183324:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.183327:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24045, 24045] 00000080:00200000:3.0:1775764896.183337:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.183340:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.183366:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98492416 i_blocks 90112 00000080:00200000:3.0:1775764896.183369:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.183383:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98492416, count: 4096 00000080:00200000:3.0:1775764896.183392:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98492416, bytes: 4096 00000080:00200000:3.0:1775764896.183394:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.183398:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24047, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.183399:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98492416, 98496512) 0 0 00000080:00200000:3.0:1775764896.183402:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.183406:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24046, 24046] 00020000:00200000:3.0:1775764896.183409:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98492416 chunk: [98492416, 98496512) 98496512, 4096 00020000:00200000:3.0:1775764896.183411:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.183421:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24046, 24046] 00000008:00010000:3.0:1775764896.183425:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.183427:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.183434:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183518:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98492416 98496511) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183528:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183535:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183543:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.183551:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98492416, 98496512) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.183556:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24046 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.183571:0:18302:0:(rw26.c:858:ll_write_end()) pos 98492416, len 4096, copied 4096 00000080:00200000:3.0:1775764896.183575:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b58cb9c9[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.183578:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003d7d731b 17ffffc0001001 3:0 ffff90c21281e460 24046 lru 00000080:00200000:3.0:1775764896.183583:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000236f863e 24046: 1< 0 - - > 2< 98492416 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2121728 - | + - + - > 5< + - + - | 0 - | 3566 - -> 00000080:00200000:3.0:1775764896.183586:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b58cb9c9 00000080:00200000:3.0:1775764896.183587:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.183588:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.183595:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.183602:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.183610:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.183617:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.183627:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.183629:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.183631:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.183639:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.183641:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.183643:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.183647:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183654:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.183663:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.183665:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.183669:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.183832:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24046, 24046] 00000080:00200000:3.0:1775764896.183843:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.183846:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.183887:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98496512 i_blocks 90112 00000080:00200000:3.0:1775764896.183891:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.183905:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98496512, count: 4096 00000080:00200000:3.0:1775764896.183915:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98496512, bytes: 4096 00000080:00200000:3.0:1775764896.183917:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.183921:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24048, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.183923:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98496512, 98500608) 0 0 00000080:00200000:3.0:1775764896.183925:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.183930:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24047, 24047] 00020000:00200000:3.0:1775764896.183933:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98496512 chunk: [98496512, 98500608) 98500608, 4096 00020000:00200000:3.0:1775764896.183935:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.183945:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24047, 24047] 00000008:00010000:3.0:1775764896.183951:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.183953:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.183961:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183970:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98496512 98500607) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183979:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183992:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.183999:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.184006:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98496512, 98500608) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.184010:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24047 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.184025:0:18302:0:(rw26.c:858:ll_write_end()) pos 98496512, len 4096, copied 4096 00000080:00200000:3.0:1775764896.184029:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000097fce095[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.184032:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003f6aa77d 17ffffc0001001 3:0 ffff90c21281ea80 24047 lru 00000080:00200000:3.0:1775764896.184037:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cf62b972 24047: 1< 0 - - > 2< 98496512 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2117632 - | + - + - > 5< + - + - | 0 - | 3567 - -> 00000080:00200000:3.0:1775764896.184039:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000097fce095 00000080:00200000:3.0:1775764896.184039:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.184041:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.184048:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184056:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184063:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184070:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.184085:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.184087:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.184090:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.184098:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.184100:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.184103:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.184106:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184127:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.184135:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.184138:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.184142:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.184144:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24047, 24047] 00000080:00200000:3.0:1775764896.184154:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.184157:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.184181:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98500608 i_blocks 90112 00000080:00200000:3.0:1775764896.184185:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.184198:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98500608, count: 4096 00000080:00200000:3.0:1775764896.184207:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98500608, bytes: 4096 00000080:00200000:3.0:1775764896.184209:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.184213:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24049, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.184214:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98500608, 98504704) 0 0 00000080:00200000:3.0:1775764896.184217:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.184222:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24048, 24048] 00020000:00200000:3.0:1775764896.184225:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98500608 chunk: [98500608, 98504704) 98504704, 4096 00020000:00200000:3.0:1775764896.184227:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.184236:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24048, 24048] 00000008:00010000:3.0:1775764896.184241:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.184243:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.184250:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184259:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98500608 98504703) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184267:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184275:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184282:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.184289:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98500608, 98504704) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.184293:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24048 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.184306:0:18302:0:(rw26.c:858:ll_write_end()) pos 98500608, len 4096, copied 4096 00000080:00200000:3.0:1775764896.184310:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000011ed15ec[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.184313:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001fd206f7 17ffffc0001001 3:0 ffff90c21281e700 24048 lru 00000080:00200000:3.0:1775764896.184318:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d142dd74 24048: 1< 0 - - > 2< 98500608 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2113536 - | + - + - > 5< + - + - | 0 - | 3568 - -> 00000080:00200000:3.0:1775764896.184320:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000011ed15ec 00000080:00200000:3.0:1775764896.184321:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.184323:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.184330:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184338:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184345:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184352:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.184362:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.184364:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.184366:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.184374:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.184376:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.184378:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.184382:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184389:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.184397:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.184399:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.184403:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.184405:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24048, 24048] 00000080:00200000:3.0:1775764896.184415:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.184418:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.184524:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98504704 i_blocks 90112 00000080:00200000:3.0:1775764896.184527:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.184543:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98504704, count: 4096 00000080:00200000:3.0:1775764896.184553:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98504704, bytes: 4096 00000080:00200000:3.0:1775764896.184556:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.184559:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24050, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.184561:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98504704, 98508800) 0 0 00000080:00200000:3.0:1775764896.184563:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.184569:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24049, 24049] 00020000:00200000:3.0:1775764896.184571:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98504704 chunk: [98504704, 98508800) 98508800, 4096 00020000:00200000:3.0:1775764896.184574:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.184584:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24049, 24049] 00000008:00010000:3.0:1775764896.184588:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.184591:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.184598:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184607:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98504704 98508799) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184615:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184622:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184629:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.184637:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98504704, 98508800) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.184641:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24049 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.184657:0:18302:0:(rw26.c:858:ll_write_end()) pos 98504704, len 4096, copied 4096 00000080:00200000:3.0:1775764896.184661:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000098435a35[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.184663:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c00bf805 17ffffc0001001 3:0 ffff90c21281ec40 24049 lru 00000080:00200000:3.0:1775764896.184669:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000024cf8560 24049: 1< 0 - - > 2< 98504704 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2109440 - | + - + - > 5< + - + - | 0 - | 3569 - -> 00000080:00200000:3.0:1775764896.184671:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000098435a35 00000080:00200000:3.0:1775764896.184672:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.184674:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.184681:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184688:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184696:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184703:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.184712:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.184715:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.184717:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.184725:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.184727:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.184729:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.184733:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184741:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.184749:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.184751:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.184754:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.184757:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24049, 24049] 00000080:00200000:3.0:1775764896.184767:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.184770:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.184794:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98508800 i_blocks 90112 00000080:00200000:3.0:1775764896.184797:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.184811:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98508800, count: 4096 00000080:00200000:3.0:1775764896.184819:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98508800, bytes: 4096 00000080:00200000:3.0:1775764896.184822:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.184825:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24051, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.184827:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98508800, 98512896) 0 0 00000080:00200000:3.0:1775764896.184829:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.184834:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24050, 24050] 00020000:00200000:3.0:1775764896.184836:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98508800 chunk: [98508800, 98512896) 98512896, 4096 00020000:00200000:3.0:1775764896.184839:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.184848:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24050, 24050] 00000008:00010000:3.0:1775764896.184852:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.184854:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.184861:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184891:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98508800 98512895) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184901:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184908:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.184916:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.184923:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98508800, 98512896) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.184927:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24050 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.184940:0:18302:0:(rw26.c:858:ll_write_end()) pos 98508800, len 4096, copied 4096 00000080:00200000:3.0:1775764896.184944:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001352864c[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.184946:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007645d362 17ffffc0001001 3:0 ffff90c21281e8c0 24050 lru 00000080:00200000:3.0:1775764896.184951:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009a6c38d8 24050: 1< 0 - - > 2< 98508800 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2105344 - | + - + - > 5< + - + - | 0 - | 3570 - -> 00000080:00200000:3.0:1775764896.184953:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001352864c 00000080:00200000:3.0:1775764896.184954:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.184956:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.184963:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184971:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184978:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.184991:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.185002:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.185004:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.185006:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.185014:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.185016:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.185019:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.185023:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185031:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.185038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.185040:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.185044:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.185046:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24050, 24050] 00000080:00200000:3.0:1775764896.185056:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.185059:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.185083:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98512896 i_blocks 90112 00000080:00200000:3.0:1775764896.185086:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.185100:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98512896, count: 4096 00000080:00200000:3.0:1775764896.185109:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98512896, bytes: 4096 00000080:00200000:3.0:1775764896.185111:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.185126:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24052, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.185128:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98512896, 98516992) 0 0 00000080:00200000:3.0:1775764896.185131:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.185136:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24051, 24051] 00020000:00200000:3.0:1775764896.185138:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98512896 chunk: [98512896, 98516992) 98516992, 4096 00020000:00200000:3.0:1775764896.185141:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.185151:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24051, 24051] 00000008:00010000:3.0:1775764896.185155:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.185157:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.185164:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185171:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98512896 98516991) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185179:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185185:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185191:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.185199:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98512896, 98516992) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.185203:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24051 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.185219:0:18302:0:(rw26.c:858:ll_write_end()) pos 98512896, len 4096, copied 4096 00000080:00200000:3.0:1775764896.185222:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a2142914[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.185225:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b19ef928 17ffffc0001001 3:0 ffff90c21281eb60 24051 lru 00000080:00200000:3.0:1775764896.185230:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000070c7b3c8 24051: 1< 0 - - > 2< 98512896 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2101248 - | + - + - > 5< + - + - | 0 - | 3571 - -> 00000080:00200000:3.0:1775764896.185232:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a2142914 00000080:00200000:3.0:1775764896.185233:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.185234:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.185241:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185248:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185254:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185260:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.185271:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.185274:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.185276:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.185285:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.185287:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.185289:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.185293:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185302:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.185309:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.185312:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.185316:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.185319:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24051, 24051] 00000080:00200000:3.0:1775764896.185329:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.185332:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.185360:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98516992 i_blocks 90112 00000080:00200000:3.0:1775764896.185364:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.185379:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98516992, count: 4096 00000080:00200000:3.0:1775764896.185389:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98516992, bytes: 4096 00000080:00200000:3.0:1775764896.185391:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.185394:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24053, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.185396:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98516992, 98521088) 0 0 00000080:00200000:3.0:1775764896.185398:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.185403:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24052, 24052] 00020000:00200000:3.0:1775764896.185406:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98516992 chunk: [98516992, 98521088) 98521088, 4096 00020000:00200000:3.0:1775764896.185408:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.185419:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24052, 24052] 00000008:00010000:3.0:1775764896.185424:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.185426:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.185434:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185507:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98516992 98521087) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185514:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185518:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185526:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.185532:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98516992, 98521088) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.185536:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24052 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.185549:0:18302:0:(rw26.c:858:ll_write_end()) pos 98516992, len 4096, copied 4096 00000080:00200000:3.0:1775764896.185551:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000036ba68d0[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.185553:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008db1697e 17ffffc0001001 3:0 ffff90c21281e000 24052 lru 00000080:00200000:3.0:1775764896.185557:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000000170b59 24052: 1< 0 - - > 2< 98516992 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2097152 - | + - + - > 5< + - + - | 0 - | 3572 - -> 00000080:00200000:3.0:1775764896.185558:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000036ba68d0 00000080:00200000:3.0:1775764896.185559:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.185560:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.185565:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185571:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185575:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185579:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.185586:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.185587:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.185589:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.185596:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.185598:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.185600:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.185602:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185607:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.185613:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.185614:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.185616:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.185618:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24052, 24052] 00000080:00200000:3.0:1775764896.185625:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.185627:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.185645:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98521088 i_blocks 90112 00000080:00200000:3.0:1775764896.185648:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.185658:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98521088, count: 4096 00000080:00200000:3.0:1775764896.185664:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98521088, bytes: 4096 00000080:00200000:3.0:1775764896.185666:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.185668:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24054, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.185669:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98521088, 98525184) 0 0 00000080:00200000:3.0:1775764896.185671:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.185674:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24053, 24053] 00020000:00200000:3.0:1775764896.185676:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98521088 chunk: [98521088, 98525184) 98525184, 4096 00020000:00200000:3.0:1775764896.185677:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.185686:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24053, 24053] 00000008:00010000:3.0:1775764896.185690:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.185691:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.185697:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185703:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98521088 98525183) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185709:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185713:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185717:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.185721:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98521088, 98525184) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.185724:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24053 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.185745:0:18302:0:(rw26.c:858:ll_write_end()) pos 98521088, len 4096, copied 4096 00000080:00200000:3.0:1775764896.185748:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000bd9ce417[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.185750:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000a552633 17ffffc0001001 3:0 ffff90c21281f7e0 24053 lru 00000080:00200000:3.0:1775764896.185754:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b41042f3 24053: 1< 0 - - > 2< 98521088 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2093056 - | + - + - > 5< + - + - | 0 - | 3573 - -> 00000080:00200000:3.0:1775764896.185755:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000bd9ce417 00000080:00200000:3.0:1775764896.185756:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.185757:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.185762:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185767:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185772:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185776:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.185783:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.185784:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.185786:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.185792:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.185793:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.185795:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.185797:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185802:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.185807:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.185808:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.185810:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.185812:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24053, 24053] 00000080:00200000:3.0:1775764896.185819:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.185821:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.185839:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98525184 i_blocks 90112 00000080:00200000:3.0:1775764896.185841:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.185852:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98525184, count: 4096 00000080:00200000:3.0:1775764896.185858:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98525184, bytes: 4096 00000080:00200000:3.0:1775764896.185859:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.185862:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24055, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.185863:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98525184, 98529280) 0 0 00000080:00200000:3.0:1775764896.185864:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.185868:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24054, 24054] 00020000:00200000:3.0:1775764896.185888:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98525184 chunk: [98525184, 98529280) 98529280, 4096 00020000:00200000:3.0:1775764896.185889:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.185897:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24054, 24054] 00000008:00010000:3.0:1775764896.185900:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.185901:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.185906:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185912:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98525184 98529279) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185917:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185922:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.185926:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.185931:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98525184, 98529280) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.185934:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24054 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.185947:0:18302:0:(rw26.c:858:ll_write_end()) pos 98525184, len 4096, copied 4096 00000080:00200000:3.0:1775764896.185952:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d2e6804b[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.185955:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000068701df7 17ffffc0001001 3:0 ffff90c21281f9a0 24054 lru 00000080:00200000:3.0:1775764896.185960:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f1574b18 24054: 1< 0 - - > 2< 98525184 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2088960 - | + - + - > 5< + - + - | 0 - | 3574 - -> 00000080:00200000:3.0:1775764896.185962:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d2e6804b 00000080:00200000:3.0:1775764896.185963:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.185965:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.185973:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185981:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185988:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.185995:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.186006:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.186009:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.186011:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.186020:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.186023:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.186025:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.186028:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186033:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.186039:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.186040:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.186043:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.186045:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24054, 24054] 00000080:00200000:3.0:1775764896.186053:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.186055:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.186074:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98529280 i_blocks 90112 00000080:00200000:3.0:1775764896.186076:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.186086:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98529280, count: 4096 00000080:00200000:3.0:1775764896.186092:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98529280, bytes: 4096 00000080:00200000:3.0:1775764896.186094:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.186096:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24056, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.186098:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98529280, 98533376) 0 0 00000080:00200000:3.0:1775764896.186099:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.186103:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24055, 24055] 00020000:00200000:3.0:1775764896.186105:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98529280 chunk: [98529280, 98533376) 98533376, 4096 00020000:00200000:3.0:1775764896.186106:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.186122:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24055, 24055] 00000008:00010000:3.0:1775764896.186128:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.186131:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.186138:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186147:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98529280 98533375) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186155:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186167:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186174:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.186182:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98529280, 98533376) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.186187:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24055 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.186205:0:18302:0:(rw26.c:858:ll_write_end()) pos 98529280, len 4096, copied 4096 00000080:00200000:3.0:1775764896.186210:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008da3cc05[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.186213:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d9418880 17ffffc0001001 3:0 ffff90c21281f2a0 24055 lru 00000080:00200000:3.0:1775764896.186218:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000143e0005 24055: 1< 0 - - > 2< 98529280 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2084864 - | + - + - > 5< + - + - | 0 - | 3575 - -> 00000080:00200000:3.0:1775764896.186220:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008da3cc05 00000080:00200000:3.0:1775764896.186221:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.186223:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.186231:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.186239:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.186245:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.186253:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.186269:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.186272:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.186274:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.186283:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.186285:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.186287:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.186291:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186299:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.186307:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.186310:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.186314:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.186316:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24055, 24055] 00000080:00200000:3.0:1775764896.186327:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.186330:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.186358:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98533376 i_blocks 90112 00000080:00200000:3.0:1775764896.186361:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.186376:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98533376, count: 4096 00000080:00200000:3.0:1775764896.186386:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98533376, bytes: 4096 00000080:00200000:3.0:1775764896.186389:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.186393:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24057, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.186395:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98533376, 98537472) 0 0 00000080:00200000:3.0:1775764896.186397:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.186402:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24056, 24056] 00020000:00200000:3.0:1775764896.186405:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98533376 chunk: [98533376, 98537472) 98537472, 4096 00020000:00200000:3.0:1775764896.186408:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.186419:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24056, 24056] 00000008:00010000:3.0:1775764896.186424:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.186426:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.186434:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186529:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98533376 98537471) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186539:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186546:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186552:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.186561:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98533376, 98537472) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.186567:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24056 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.186592:0:18302:0:(rw26.c:858:ll_write_end()) pos 98533376, len 4096, copied 4096 00000080:00200000:3.0:1775764896.186596:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e080f0f0[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.186599:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000061aaec7a 17ffffc0001021 2:0 ffff90c21281fd20 24056 lru 00000080:00200000:3.0:1775764896.186605:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000742b5420 24056: 1< 0 - - > 2< 98533376 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2080768 - | + - + - > 5< + - + - | 0 - | 3576 - -> 00000080:00200000:3.0:1775764896.186607:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e080f0f0 00000080:00200000:3.0:1775764896.186608:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.186610:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.186619:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.186628:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.186635:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.186643:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.186655:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.186658:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.186660:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.186671:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.186674:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.186677:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.186681:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186689:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.186698:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.186700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.186704:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.186707:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24056, 24056] 00000080:00200000:3.0:1775764896.186719:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.186722:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.186753:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98537472 i_blocks 90112 00000080:00200000:3.0:1775764896.186756:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.186773:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98537472, count: 4096 00000080:00200000:3.0:1775764896.186784:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98537472, bytes: 4096 00000080:00200000:3.0:1775764896.186786:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.186790:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24058, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.186792:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98537472, 98541568) 0 0 00000080:00200000:3.0:1775764896.186794:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.186800:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24057, 24057] 00020000:00200000:3.0:1775764896.186803:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98537472 chunk: [98537472, 98541568) 98541568, 4096 00020000:00200000:3.0:1775764896.186805:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.186818:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24057, 24057] 00000008:00010000:3.0:1775764896.186823:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.186826:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.186834:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186843:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98537472 98541567) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186851:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186859:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.186866:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.186897:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98537472, 98541568) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.186903:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24057 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.186921:0:18302:0:(rw26.c:858:ll_write_end()) pos 98537472, len 4096, copied 4096 00000080:00200000:3.0:1775764896.186925:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b5d5dbad[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.186928:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005604aa79 17ffffc0001001 3:0 ffff90c21281f380 24057 lru 00000080:00200000:3.0:1775764896.186934:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cf135e84 24057: 1< 0 - - > 2< 98537472 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2076672 - | + - + - > 5< + - + - | 0 - | 3577 - -> 00000080:00200000:3.0:1775764896.186936:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b5d5dbad 00000080:00200000:3.0:1775764896.186937:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.186939:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.186947:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.186955:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.186962:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.186971:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.186982:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.186985:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.186987:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.186998:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.187000:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.187003:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.187007:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187015:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.187023:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.187026:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.187030:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.187033:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24057, 24057] 00000080:00200000:3.0:1775764896.187044:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.187047:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.187076:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98541568 i_blocks 90112 00000080:00200000:3.0:1775764896.187079:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.187094:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98541568, count: 4096 00000080:00200000:3.0:1775764896.187104:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98541568, bytes: 4096 00000080:00200000:3.0:1775764896.187107:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.187110:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24059, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.187112:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98541568, 98545664) 0 0 00000080:00200000:3.0:1775764896.187126:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.187132:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24058, 24058] 00020000:00200000:3.0:1775764896.187135:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98541568 chunk: [98541568, 98545664) 98545664, 4096 00020000:00200000:3.0:1775764896.187137:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.187150:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24058, 24058] 00000008:00010000:3.0:1775764896.187156:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.187158:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.187166:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187175:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98541568 98545663) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187184:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187192:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187199:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.187207:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98541568, 98545664) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.187213:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24058 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.187229:0:18302:0:(rw26.c:858:ll_write_end()) pos 98541568, len 4096, copied 4096 00000080:00200000:3.0:1775764896.187233:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007c84ef3a[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.187236:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006f6f8fea 17ffffc0001001 3:0 ffff90c21281f620 24058 lru 00000080:00200000:3.0:1775764896.187242:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cf692cd2 24058: 1< 0 - - > 2< 98541568 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2072576 - | + - + - > 5< + - + - | 0 - | 3578 - -> 00000080:00200000:3.0:1775764896.187244:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007c84ef3a 00000080:00200000:3.0:1775764896.187245:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.187247:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.187255:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.187517:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.187525:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.187538:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.187552:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.187556:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.187558:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.187569:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.187571:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.187574:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.187579:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187587:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.187596:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.187598:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.187603:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.187605:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24058, 24058] 00000080:00200000:3.0:1775764896.187616:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.187619:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.187648:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98545664 i_blocks 90112 00000080:00200000:3.0:1775764896.187651:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.187665:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98545664, count: 4096 00000080:00200000:3.0:1775764896.187676:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98545664, bytes: 4096 00000080:00200000:3.0:1775764896.187678:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.187682:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24060, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.187684:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98545664, 98549760) 0 0 00000080:00200000:3.0:1775764896.187686:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.187692:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24059, 24059] 00020000:00200000:3.0:1775764896.187695:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98545664 chunk: [98545664, 98549760) 98549760, 4096 00020000:00200000:3.0:1775764896.187697:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.187708:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24059, 24059] 00000008:00010000:3.0:1775764896.187714:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.187717:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.187725:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187734:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98545664 98549759) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187742:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187749:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187756:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.187764:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98545664, 98549760) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.187770:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24059 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.187792:0:18302:0:(rw26.c:858:ll_write_end()) pos 98545664, len 4096, copied 4096 00000080:00200000:3.0:1775764896.187796:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008b95dbfe[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.187798:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ec06eb83 17ffffc0001001 3:0 ffff90c21281fe00 24059 lru 00000080:00200000:3.0:1775764896.187804:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a8dc8ff4 24059: 1< 0 - - > 2< 98545664 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2068480 - | + - + - > 5< + - + - | 0 - | 3579 - -> 00000080:00200000:3.0:1775764896.187806:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008b95dbfe 00000080:00200000:3.0:1775764896.187807:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.187809:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.187817:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.187824:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.187831:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.187838:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.187849:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.187851:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.187854:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.187864:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.187866:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.187971:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.187977:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.187985:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.187994:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.187997:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.188002:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.188005:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24059, 24059] 00000080:00200000:3.0:1775764896.188015:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.188019:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.188052:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98549760 i_blocks 90112 00000080:00200000:3.0:1775764896.188056:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.188073:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98549760, count: 4096 00000080:00200000:3.0:1775764896.188083:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98549760, bytes: 4096 00000080:00200000:3.0:1775764896.188086:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.188090:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24061, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.188091:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98549760, 98553856) 0 0 00000080:00200000:3.0:1775764896.188094:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.188099:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24060, 24060] 00020000:00200000:3.0:1775764896.188102:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98549760 chunk: [98549760, 98553856) 98553856, 4096 00020000:00200000:3.0:1775764896.188104:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.188129:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24060, 24060] 00000008:00010000:3.0:1775764896.188135:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.188137:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.188146:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188156:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98549760 98553855) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188165:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188172:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188185:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.188193:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98549760, 98553856) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.188198:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24060 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.188216:0:18302:0:(rw26.c:858:ll_write_end()) pos 98549760, len 4096, copied 4096 00000080:00200000:3.0:1775764896.188220:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000035dda41f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.188223:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ee4495b6 17ffffc0001001 3:0 ffff90c21281f540 24060 lru 00000080:00200000:3.0:1775764896.188229:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f2abb850 24060: 1< 0 - - > 2< 98549760 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2064384 - | + - + - > 5< + - + - | 0 - | 3580 - -> 00000080:00200000:3.0:1775764896.188231:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000035dda41f 00000080:00200000:3.0:1775764896.188233:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.188235:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.188243:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.188251:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.188258:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.188266:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.188278:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.188281:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.188284:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.188294:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.188297:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.188299:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.188303:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188311:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.188320:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.188322:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.188326:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.188329:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24060, 24060] 00000080:00200000:3.0:1775764896.188339:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.188342:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.188370:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98553856 i_blocks 90112 00000080:00200000:3.0:1775764896.188374:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.188390:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98553856, count: 4096 00000080:00200000:3.0:1775764896.188400:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98553856, bytes: 4096 00000080:00200000:3.0:1775764896.188403:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.188406:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24062, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.188408:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98553856, 98557952) 0 0 00000080:00200000:3.0:1775764896.188410:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.188415:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24061, 24061] 00020000:00200000:3.0:1775764896.188418:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98553856 chunk: [98553856, 98557952) 98557952, 4096 00020000:00200000:3.0:1775764896.188420:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.188431:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24061, 24061] 00000008:00010000:3.0:1775764896.188436:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.188514:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.188524:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188533:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98553856 98557951) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188541:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188548:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188554:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.188561:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98553856, 98557952) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.188568:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24061 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.188588:0:18302:0:(rw26.c:858:ll_write_end()) pos 98553856, len 4096, copied 4096 00000080:00200000:3.0:1775764896.188593:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000281d2391[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.188595:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003421d289 17ffffc0001001 3:0 ffff90c21281f460 24061 lru 00000080:00200000:3.0:1775764896.188599:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b76b6f8b 24061: 1< 0 - - > 2< 98553856 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2060288 - | + - + - > 5< + - + - | 0 - | 3581 - -> 00000080:00200000:3.0:1775764896.188601:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000281d2391 00000080:00200000:3.0:1775764896.188602:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.188604:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.188612:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.188619:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.188626:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.188632:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.188642:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.188645:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.188647:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.188657:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.188660:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.188662:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.188665:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188672:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.188680:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.188681:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.188686:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.188688:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24061, 24061] 00000080:00200000:3.0:1775764896.188698:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.188701:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.188730:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98557952 i_blocks 90112 00000080:00200000:3.0:1775764896.188734:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.188750:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98557952, count: 4096 00000080:00200000:3.0:1775764896.188760:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98557952, bytes: 4096 00000080:00200000:3.0:1775764896.188763:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.188767:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24063, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.188768:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98557952, 98562048) 0 0 00000080:00200000:3.0:1775764896.188771:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.188776:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24062, 24062] 00020000:00200000:3.0:1775764896.188779:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98557952 chunk: [98557952, 98562048) 98562048, 4096 00020000:00200000:3.0:1775764896.188781:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.188792:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24062, 24062] 00000008:00010000:3.0:1775764896.188798:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.188800:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.188808:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188816:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98557952 98562047) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188823:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188829:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188836:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.188843:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98557952, 98562048) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.188848:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24062 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.188864:0:18302:0:(rw26.c:858:ll_write_end()) pos 98557952, len 4096, copied 4096 00000080:00200000:3.0:1775764896.188867:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ae984d4e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.188890:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f7823aaa 17ffffc0001001 3:0 ffff90c21281fa80 24062 lru 00000080:00200000:3.0:1775764896.188895:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000041659c43 24062: 1< 0 - - > 2< 98557952 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2056192 - | + - + - > 5< + - + - | 0 - | 3582 - -> 00000080:00200000:3.0:1775764896.188897:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ae984d4e 00000080:00200000:3.0:1775764896.188898:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.188901:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.188908:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.188916:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.188922:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.188929:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.188941:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.188943:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.188946:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.188955:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.188958:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.188960:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.188964:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.188972:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.188983:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.188985:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.188989:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.188992:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24062, 24062] 00000080:00200000:3.0:1775764896.189002:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.189005:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.189034:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98562048 i_blocks 90112 00000080:00200000:3.0:1775764896.189038:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.189054:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98562048, count: 4096 00000080:00200000:3.0:1775764896.189064:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98562048, bytes: 4096 00000080:00200000:3.0:1775764896.189067:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.189070:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24064, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.189072:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98562048, 98566144) 0 0 00000080:00200000:3.0:1775764896.189074:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.189080:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24063, 24063] 00020000:00200000:3.0:1775764896.189083:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98562048 chunk: [98562048, 98566144) 98566144, 4096 00020000:00200000:3.0:1775764896.189085:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.189096:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24063, 24063] 00000008:00010000:3.0:1775764896.189102:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.189104:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.189111:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189131:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98562048 98566143) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189139:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189151:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189157:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.189163:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98562048, 98566144) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.189168:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24063 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.189183:0:18302:0:(rw26.c:858:ll_write_end()) pos 98562048, len 4096, copied 4096 00000080:00200000:3.0:1775764896.189186:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000273d4510[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.189189:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000634855c5 17ffffc0001001 3:0 ffff90c21281f700 24063 lru 00000080:00200000:3.0:1775764896.189194:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000184a0a1a 24063: 1< 0 - - > 2< 98562048 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2052096 - | + - + - > 5< + - + - | 0 - | 3583 - -> 00000080:00200000:3.0:1775764896.189196:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000273d4510 00000080:00200000:3.0:1775764896.189197:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.189199:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.189206:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.189213:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.189220:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.189227:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.189244:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.189247:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.189249:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.189259:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.189262:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.189264:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.189268:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189276:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.189284:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.189287:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.189291:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.189294:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24063, 24063] 00000080:00200000:3.0:1775764896.189305:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.189308:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.189338:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98566144 i_blocks 90112 00000080:00200000:3.0:1775764896.189342:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.189359:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98566144, count: 4096 00000080:00200000:3.0:1775764896.189369:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98566144, bytes: 4096 00000080:00200000:3.0:1775764896.189371:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.189374:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24065, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.189375:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98566144, 98570240) 0 0 00000080:00200000:3.0:1775764896.189377:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.189382:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24064, 24064] 00020000:00200000:3.0:1775764896.189386:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98566144 chunk: [98566144, 98570240) 98570240, 4096 00020000:00200000:3.0:1775764896.189387:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.189399:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24064, 24064] 00000008:00010000:3.0:1775764896.189404:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.189406:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.189414:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189423:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98566144 98570239) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189430:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189511:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189520:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.189529:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98566144, 98570240) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.189535:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24064 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.189557:0:18302:0:(rw26.c:858:ll_write_end()) pos 98566144, len 4096, copied 4096 00000080:00200000:3.0:1775764896.189562:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000002a80884[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.189564:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c7ca2aca 17ffffc0001001 3:0 ffff90c21281fc40 24064 lru 00000080:00200000:3.0:1775764896.189570:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000059054b97 24064: 1< 0 - - > 2< 98566144 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2048000 - | + - + - > 5< + - + - | 0 - | 3584 - -> 00000080:00200000:3.0:1775764896.189572:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000002a80884 00000080:00200000:3.0:1775764896.189573:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.189575:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.189583:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.189591:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.189598:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.189606:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.189617:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.189620:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.189623:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.189633:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.189636:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.189638:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.189642:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189649:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.189658:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.189661:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.189665:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.189667:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24064, 24064] 00000080:00200000:3.0:1775764896.189679:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.189682:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.189713:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98570240 i_blocks 90112 00000080:00200000:3.0:1775764896.189716:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.189731:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98570240, count: 4096 00000080:00200000:3.0:1775764896.189739:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98570240, bytes: 4096 00000080:00200000:3.0:1775764896.189742:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.189745:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24066, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.189746:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98570240, 98574336) 0 0 00000080:00200000:3.0:1775764896.189748:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.189753:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24065, 24065] 00020000:00200000:3.0:1775764896.189756:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98570240 chunk: [98570240, 98574336) 98574336, 4096 00020000:00200000:3.0:1775764896.189758:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.189767:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24065, 24065] 00000008:00010000:3.0:1775764896.189772:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.189774:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.189781:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189790:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98570240 98574335) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189799:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189806:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189813:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.189821:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98570240, 98574336) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.189827:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24065 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.189843:0:18302:0:(rw26.c:858:ll_write_end()) pos 98570240, len 4096, copied 4096 00000080:00200000:3.0:1775764896.189847:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000199e6949[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.189850:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000dd7bdb90 17ffffc0001001 3:0 ffff90c21281f8c0 24065 lru 00000080:00200000:3.0:1775764896.189855:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000008cd0d82 24065: 1< 0 - - > 2< 98570240 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2043904 - | + - + - > 5< + - + - | 0 - | 3585 - -> 00000080:00200000:3.0:1775764896.189857:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000199e6949 00000080:00200000:3.0:1775764896.189859:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.189861:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.189868:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.189897:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.189903:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.189911:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.189922:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.189925:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.189927:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.189936:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.189939:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.189941:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.189946:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.189954:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.189962:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.189965:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.189969:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.189972:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24065, 24065] 00000080:00200000:3.0:1775764896.189982:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.189986:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.190016:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98574336 i_blocks 90112 00000080:00200000:3.0:1775764896.190020:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.190036:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98574336, count: 4096 00000080:00200000:3.0:1775764896.190047:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98574336, bytes: 4096 00000080:00200000:3.0:1775764896.190050:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.190053:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24067, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.190055:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98574336, 98578432) 0 0 00000080:00200000:3.0:1775764896.190057:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.190062:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24066, 24066] 00020000:00200000:3.0:1775764896.190066:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98574336 chunk: [98574336, 98578432) 98578432, 4096 00020000:00200000:3.0:1775764896.190068:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.190080:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24066, 24066] 00000008:00010000:3.0:1775764896.190085:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.190087:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.190096:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190106:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98574336 98578431) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190126:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190133:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190140:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.190148:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98574336, 98578432) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.190153:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24066 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.190169:0:18302:0:(rw26.c:858:ll_write_end()) pos 98574336, len 4096, copied 4096 00000080:00200000:3.0:1775764896.190174:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000638d56e9[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.190177:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000166c8ad0 17ffffc0001001 3:0 ffff90c21281fb60 24066 lru 00000080:00200000:3.0:1775764896.190182:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000dec8bf06 24066: 1< 0 - - > 2< 98574336 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2039808 - | + - + - > 5< + - + - | 0 - | 3586 - -> 00000080:00200000:3.0:1775764896.190185:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000638d56e9 00000080:00200000:3.0:1775764896.190186:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.190188:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.190195:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.190202:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.190209:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.190222:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.190234:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.190236:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.190239:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.190249:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.190252:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.190254:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.190258:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190267:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.190275:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.190277:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.190282:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.190286:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24066, 24066] 00000080:00200000:3.0:1775764896.190297:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.190300:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.190330:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98578432 i_blocks 90112 00000080:00200000:3.0:1775764896.190334:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.190349:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98578432, count: 4096 00000080:00200000:3.0:1775764896.190359:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98578432, bytes: 4096 00000080:00200000:3.0:1775764896.190362:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.190365:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24068, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.190367:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98578432, 98582528) 0 0 00000080:00200000:3.0:1775764896.190370:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.190374:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24067, 24067] 00020000:00200000:3.0:1775764896.190377:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98578432 chunk: [98578432, 98582528) 98582528, 4096 00020000:00200000:3.0:1775764896.190380:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.190391:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24067, 24067] 00000008:00010000:3.0:1775764896.190396:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.190399:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.190407:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190415:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98578432 98582527) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190423:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190430:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190509:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.190520:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98578432, 98582528) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.190525:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24067 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.190541:0:18302:0:(rw26.c:858:ll_write_end()) pos 98578432, len 4096, copied 4096 00000080:00200000:3.0:1775764896.190545:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005ee0dc5e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.190548:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000974bf3dc 17ffffc0001001 3:0 ffff90c21281f000 24067 lru 00000080:00200000:3.0:1775764896.190553:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e1053e72 24067: 1< 0 - - > 2< 98578432 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2035712 - | + - + - > 5< + - + - | 0 - | 3587 - -> 00000080:00200000:3.0:1775764896.190555:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005ee0dc5e 00000080:00200000:3.0:1775764896.190556:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.190558:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.190566:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.190572:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.190579:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.190587:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.190598:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.190600:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.190603:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.190612:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.190614:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.190617:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.190621:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190629:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.190638:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.190640:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.190644:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.190647:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24067, 24067] 00000080:00200000:3.0:1775764896.190658:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.190661:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.190687:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98582528 i_blocks 90112 00000080:00200000:3.0:1775764896.190691:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.190705:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98582528, count: 4096 00000080:00200000:3.0:1775764896.190715:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98582528, bytes: 4096 00000080:00200000:3.0:1775764896.190718:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.190721:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24069, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.190723:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98582528, 98586624) 0 0 00000080:00200000:3.0:1775764896.190725:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.190731:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24068, 24068] 00020000:00200000:3.0:1775764896.190734:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98582528 chunk: [98582528, 98586624) 98586624, 4096 00020000:00200000:3.0:1775764896.190736:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.190748:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24068, 24068] 00000008:00010000:3.0:1775764896.190753:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.190755:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.190762:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190771:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98582528 98586623) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190779:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190787:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190798:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.190806:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98582528, 98586624) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.190811:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24068 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.190827:0:18302:0:(rw26.c:858:ll_write_end()) pos 98582528, len 4096, copied 4096 00000080:00200000:3.0:1775764896.190831:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007cd3b8fd[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.190834:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001cefc426 17ffffc0001001 3:0 ffff90c21281fee0 24068 lru 00000080:00200000:3.0:1775764896.190839:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001665990c 24068: 1< 0 - - > 2< 98582528 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2031616 - | + - + - > 5< + - + - | 0 - | 3588 - -> 00000080:00200000:3.0:1775764896.190841:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007cd3b8fd 00000080:00200000:3.0:1775764896.190843:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.190845:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.190852:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.190860:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.190867:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.190896:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.190908:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.190911:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.190914:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.190923:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.190925:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.190927:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.190931:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.190937:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.190945:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.190947:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.190951:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.190954:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24068, 24068] 00000080:00200000:3.0:1775764896.190966:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.190969:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.190997:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98586624 i_blocks 90112 00000080:00200000:3.0:1775764896.191000:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.191016:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98586624, count: 4096 00000080:00200000:3.0:1775764896.191025:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98586624, bytes: 4096 00000080:00200000:3.0:1775764896.191027:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.191031:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24070, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.191032:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98586624, 98590720) 0 0 00000080:00200000:3.0:1775764896.191034:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.191038:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24069, 24069] 00020000:00200000:3.0:1775764896.191041:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98586624 chunk: [98586624, 98590720) 98590720, 4096 00020000:00200000:3.0:1775764896.191043:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.191054:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24069, 24069] 00000008:00010000:3.0:1775764896.191058:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.191061:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.191068:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191077:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98586624 98590719) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191085:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191091:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191097:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.191105:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98586624, 98590720) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.191111:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24069 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.191138:0:18302:0:(rw26.c:858:ll_write_end()) pos 98586624, len 4096, copied 4096 00000080:00200000:3.0:1775764896.191143:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b54ac81f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.191146:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000223d3c9a 17ffffc0001001 3:0 ffff90c21281f0e0 24069 lru 00000080:00200000:3.0:1775764896.191151:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000bd201fdb 24069: 1< 0 - - > 2< 98586624 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2027520 - | + - + - > 5< + - + - | 0 - | 3589 - -> 00000080:00200000:3.0:1775764896.191153:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b54ac81f 00000080:00200000:3.0:1775764896.191154:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.191156:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.191164:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.191172:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.191179:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.191186:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.191197:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.191200:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.191202:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.191211:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.191214:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.191217:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.191221:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191229:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.191236:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.191238:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.191242:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.191245:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24069, 24069] 00000080:00200000:3.0:1775764896.191256:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.191259:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.191288:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98590720 i_blocks 90112 00000080:00200000:3.0:1775764896.191292:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.191308:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98590720, count: 4096 00000080:00200000:3.0:1775764896.191319:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98590720, bytes: 4096 00000080:00200000:3.0:1775764896.191322:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.191325:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24071, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.191327:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98590720, 98594816) 0 0 00000080:00200000:3.0:1775764896.191329:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.191334:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24070, 24070] 00020000:00200000:3.0:1775764896.191337:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98590720 chunk: [98590720, 98594816) 98594816, 4096 00020000:00200000:3.0:1775764896.191339:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.191351:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24070, 24070] 00000008:00010000:3.0:1775764896.191356:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.191358:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.191366:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191374:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98590720 98594815) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191383:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191391:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191398:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.191406:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98590720, 98594816) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.191411:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24070 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.191430:0:18302:0:(rw26.c:858:ll_write_end()) pos 98590720, len 4096, copied 4096 00000080:00200000:3.0:1775764896.191434:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000be002404[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.191515:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000066cd705e 17ffffc0001001 3:0 ffff90c21281f1c0 24070 lru 00000080:00200000:3.0:1775764896.191522:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d676e59d 24070: 1< 0 - - > 2< 98590720 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2023424 - | + - + - > 5< + - + - | 0 - | 3590 - -> 00000080:00200000:3.0:1775764896.191524:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000be002404 00000080:00200000:3.0:1775764896.191525:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.191527:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.191537:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.191544:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.191551:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.191557:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.191568:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.191571:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.191573:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.191582:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.191585:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.191588:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.191592:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191599:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.191607:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.191610:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.191613:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.191616:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24070, 24070] 00000080:00200000:3.0:1775764896.191627:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.191630:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.191661:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98594816 i_blocks 90112 00000080:00200000:3.0:1775764896.191665:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.191680:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98594816, count: 4096 00000080:00200000:3.0:1775764896.191690:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98594816, bytes: 4096 00000080:00200000:3.0:1775764896.191693:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.191696:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24072, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.191698:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98594816, 98598912) 0 0 00000080:00200000:3.0:1775764896.191700:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.191705:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24071, 24071] 00020000:00200000:3.0:1775764896.191708:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98594816 chunk: [98594816, 98598912) 98598912, 4096 00020000:00200000:3.0:1775764896.191710:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.191722:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24071, 24071] 00000008:00010000:3.0:1775764896.191728:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.191730:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.191738:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191747:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98594816 98598911) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191756:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191769:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191776:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.191784:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98594816, 98598912) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.191789:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24071 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.191825:0:18302:0:(rw26.c:858:ll_write_end()) pos 98594816, len 4096, copied 4096 00000080:00200000:3.0:1775764896.191829:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004991c865[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.191832:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d786597a 17ffffc0001021 2:0 ffff90c2128209a0 24071 lru 00000080:00200000:3.0:1775764896.191838:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d9bf66ac 24071: 1< 0 - - > 2< 98594816 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2019328 - | + - + - > 5< + - + - | 0 - | 3591 - -> 00000080:00200000:3.0:1775764896.191839:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004991c865 00000080:00200000:3.0:1775764896.191840:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.191843:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.191851:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.191859:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.191865:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.191895:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.191913:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.191916:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.191918:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.191929:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.191931:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.191934:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.191938:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.191946:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.191955:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.191957:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.191962:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.191965:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24071, 24071] 00000080:00200000:3.0:1775764896.191976:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.191979:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.192009:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98598912 i_blocks 90112 00000080:00200000:3.0:1775764896.192012:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.192028:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98598912, count: 4096 00000080:00200000:3.0:1775764896.192038:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98598912, bytes: 4096 00000080:00200000:3.0:1775764896.192040:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.192044:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24073, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.192045:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98598912, 98603008) 0 0 00000080:00200000:3.0:1775764896.192047:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.192052:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24072, 24072] 00020000:00200000:3.0:1775764896.192055:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98598912 chunk: [98598912, 98603008) 98603008, 4096 00020000:00200000:3.0:1775764896.192058:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.192068:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24072, 24072] 00000008:00010000:3.0:1775764896.192073:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.192076:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.192084:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192093:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98598912 98603007) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192101:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192108:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192126:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.192135:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98598912, 98603008) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.192141:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24072 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.192158:0:18302:0:(rw26.c:858:ll_write_end()) pos 98598912, len 4096, copied 4096 00000080:00200000:3.0:1775764896.192162:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b811d954[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.192165:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008586a82d 17ffffc0001001 3:0 ffff90c2128202a0 24072 lru 00000080:00200000:3.0:1775764896.192170:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000010e15c2b 24072: 1< 0 - - > 2< 98598912 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2015232 - | + - + - > 5< + - + - | 0 - | 3592 - -> 00000080:00200000:3.0:1775764896.192173:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b811d954 00000080:00200000:3.0:1775764896.192174:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.192176:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.192185:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.192192:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.192199:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.192207:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.192219:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.192222:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.192224:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.192234:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.192237:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.192239:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.192243:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192250:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.192258:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.192260:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.192264:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.192267:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24072, 24072] 00000080:00200000:3.0:1775764896.192278:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.192281:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.192311:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98603008 i_blocks 90112 00000080:00200000:3.0:1775764896.192315:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.192330:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98603008, count: 4096 00000080:00200000:3.0:1775764896.192341:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98603008, bytes: 4096 00000080:00200000:3.0:1775764896.192343:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.192347:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24074, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.192348:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98603008, 98607104) 0 0 00000080:00200000:3.0:1775764896.192350:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.192356:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24073, 24073] 00020000:00200000:3.0:1775764896.192360:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98603008 chunk: [98603008, 98607104) 98607104, 4096 00020000:00200000:3.0:1775764896.192362:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.192373:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24073, 24073] 00000008:00010000:3.0:1775764896.192378:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.192380:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.192388:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192397:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98603008 98607103) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192406:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192414:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192420:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.192429:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98603008, 98607104) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.192434:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24073 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.192541:0:18302:0:(rw26.c:858:ll_write_end()) pos 98603008, len 4096, copied 4096 00000080:00200000:3.0:1775764896.192546:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000178722f0[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.192549:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f446af37 17ffffc0001001 3:0 ffff90c212820d20 24073 lru 00000080:00200000:3.0:1775764896.192554:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000989a3527 24073: 1< 0 - - > 2< 98603008 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2011136 - | + - + - > 5< + - + - | 0 - | 3593 - -> 00000080:00200000:3.0:1775764896.192557:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000178722f0 00000080:00200000:3.0:1775764896.192558:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.192561:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.192569:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.192578:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.192585:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.192592:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.192603:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.192612:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.192614:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.192626:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.192628:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.192631:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.192636:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192644:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.192651:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.192653:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.192657:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.192660:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24073, 24073] 00000080:00200000:3.0:1775764896.192671:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.192674:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.192705:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98607104 i_blocks 90112 00000080:00200000:3.0:1775764896.192709:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.192725:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98607104, count: 4096 00000080:00200000:3.0:1775764896.192736:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98607104, bytes: 4096 00000080:00200000:3.0:1775764896.192739:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.192743:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24075, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.192744:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98607104, 98611200) 0 0 00000080:00200000:3.0:1775764896.192747:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.192752:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24074, 24074] 00020000:00200000:3.0:1775764896.192755:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98607104 chunk: [98607104, 98611200) 98611200, 4096 00020000:00200000:3.0:1775764896.192757:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.192767:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24074, 24074] 00000008:00010000:3.0:1775764896.192772:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.192774:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.192782:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192791:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98607104 98611199) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192799:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192806:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192813:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.192820:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98607104, 98611200) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.192826:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24074 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.192842:0:18302:0:(rw26.c:858:ll_write_end()) pos 98607104, len 4096, copied 4096 00000080:00200000:3.0:1775764896.192846:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008214eb23[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.192849:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f278fad5 17ffffc0001001 3:0 ffff90c212820380 24074 lru 00000080:00200000:3.0:1775764896.192855:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008e3faf78 24074: 1< 0 - - > 2< 98607104 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2007040 - | + - + - > 5< + - + - | 0 - | 3594 - -> 00000080:00200000:3.0:1775764896.192857:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008214eb23 00000080:00200000:3.0:1775764896.192858:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.192860:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.192868:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.192898:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.192905:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.192919:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.192931:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.192934:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.192936:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.192946:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.192949:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.192952:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.192956:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.192964:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.192972:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.192974:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.192979:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.192982:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24074, 24074] 00000080:00200000:3.0:1775764896.192993:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.192996:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.193025:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98611200 i_blocks 90112 00000080:00200000:3.0:1775764896.193029:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.193045:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98611200, count: 4096 00000080:00200000:3.0:1775764896.193055:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98611200, bytes: 4096 00000080:00200000:3.0:1775764896.193058:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.193063:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24076, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.193064:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98611200, 98615296) 0 0 00000080:00200000:3.0:1775764896.193067:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.193072:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24075, 24075] 00020000:00200000:3.0:1775764896.193075:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98611200 chunk: [98611200, 98615296) 98615296, 4096 00020000:00200000:3.0:1775764896.193078:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.193089:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24075, 24075] 00000008:00010000:3.0:1775764896.193094:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.193097:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.193104:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193123:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98611200 98615295) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193132:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193139:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193146:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.193153:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98611200, 98615296) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.193158:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24075 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.193176:0:18302:0:(rw26.c:858:ll_write_end()) pos 98611200, len 4096, copied 4096 00000080:00200000:3.0:1775764896.193180:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004811f6ed[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.193183:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000bb11a43 17ffffc0001001 3:0 ffff90c212820620 24075 lru 00000080:00200000:3.0:1775764896.193189:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f7bc0e37 24075: 1< 0 - - > 2< 98611200 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 2002944 - | + - + - > 5< + - + - | 0 - | 3595 - -> 00000080:00200000:3.0:1775764896.193192:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004811f6ed 00000080:00200000:3.0:1775764896.193193:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.193195:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.193203:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.193211:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.193217:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.193224:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.193235:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.193238:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.193241:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.193250:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.193252:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.193254:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.193258:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193266:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.193274:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.193277:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.193281:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.193283:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24075, 24075] 00000080:00200000:3.0:1775764896.193293:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.193296:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.193322:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98615296 i_blocks 90112 00000080:00200000:3.0:1775764896.193326:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.193341:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98615296, count: 4096 00000080:00200000:3.0:1775764896.193351:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98615296, bytes: 4096 00000080:00200000:3.0:1775764896.193353:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.193357:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24077, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.193358:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98615296, 98619392) 0 0 00000080:00200000:3.0:1775764896.193361:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.193366:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24076, 24076] 00020000:00200000:3.0:1775764896.193368:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98615296 chunk: [98615296, 98619392) 98619392, 4096 00020000:00200000:3.0:1775764896.193370:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.193382:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24076, 24076] 00000008:00010000:3.0:1775764896.193386:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.193389:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.193396:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193406:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98615296 98619391) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193414:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193422:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193435:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.193521:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98615296, 98619392) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.193526:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24076 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.193544:0:18302:0:(rw26.c:858:ll_write_end()) pos 98615296, len 4096, copied 4096 00000080:00200000:3.0:1775764896.193548:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fd0fa313[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.193551:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007cac19e9 17ffffc0001001 3:0 ffff90c212820e00 24076 lru 00000080:00200000:3.0:1775764896.193557:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004d526e7d 24076: 1< 0 - - > 2< 98615296 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1998848 - | + - + - > 5< + - + - | 0 - | 3596 - -> 00000080:00200000:3.0:1775764896.193559:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fd0fa313 00000080:00200000:3.0:1775764896.193561:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.193563:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.193570:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.193577:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.193583:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.193589:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.193599:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.193602:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.193604:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.193614:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.193617:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.193619:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.193624:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193631:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.193638:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.193641:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.193644:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.193647:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24076, 24076] 00000080:00200000:3.0:1775764896.193657:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.193660:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.193685:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98619392 i_blocks 90112 00000080:00200000:3.0:1775764896.193688:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.193702:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98619392, count: 4096 00000080:00200000:3.0:1775764896.193711:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98619392, bytes: 4096 00000080:00200000:3.0:1775764896.193713:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.193716:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24078, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.193718:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98619392, 98623488) 0 0 00000080:00200000:3.0:1775764896.193720:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.193725:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24077, 24077] 00020000:00200000:3.0:1775764896.193727:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98619392 chunk: [98619392, 98623488) 98623488, 4096 00020000:00200000:3.0:1775764896.193729:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.193738:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24077, 24077] 00000008:00010000:3.0:1775764896.193743:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.193745:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.193753:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193760:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98619392 98623487) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193768:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193775:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193783:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.193791:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98619392, 98623488) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.193795:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24077 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.193812:0:18302:0:(rw26.c:858:ll_write_end()) pos 98619392, len 4096, copied 4096 00000080:00200000:3.0:1775764896.193815:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f4935416[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.193818:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000086842230 17ffffc0001001 3:0 ffff90c212820540 24077 lru 00000080:00200000:3.0:1775764896.193824:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005754295d 24077: 1< 0 - - > 2< 98619392 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1994752 - | + - + - > 5< + - + - | 0 - | 3597 - -> 00000080:00200000:3.0:1775764896.193826:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f4935416 00000080:00200000:3.0:1775764896.193827:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.193829:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.193836:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.193844:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.193852:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.193859:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.193891:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.193895:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.193897:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.193906:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.193909:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.193911:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.193914:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.193922:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.193929:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.193931:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.193935:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.193938:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24077, 24077] 00000080:00200000:3.0:1775764896.193949:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.193952:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.193982:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98623488 i_blocks 90112 00000080:00200000:3.0:1775764896.193985:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.194001:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98623488, count: 4096 00000080:00200000:3.0:1775764896.194011:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98623488, bytes: 4096 00000080:00200000:3.0:1775764896.194014:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.194017:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24079, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.194019:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98623488, 98627584) 0 0 00000080:00200000:3.0:1775764896.194021:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.194027:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24078, 24078] 00020000:00200000:3.0:1775764896.194029:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98623488 chunk: [98623488, 98627584) 98627584, 4096 00020000:00200000:3.0:1775764896.194032:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.194043:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24078, 24078] 00000008:00010000:3.0:1775764896.194048:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.194051:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.194058:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194067:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98623488 98627583) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194075:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194083:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194090:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.194097:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98623488, 98627584) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.194102:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24078 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.194129:0:18302:0:(rw26.c:858:ll_write_end()) pos 98623488, len 4096, copied 4096 00000080:00200000:3.0:1775764896.194133:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000f2f4eb2[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.194137:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000be97f3b 17ffffc0001001 3:0 ffff90c212820460 24078 lru 00000080:00200000:3.0:1775764896.194142:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000005fbf46d 24078: 1< 0 - - > 2< 98623488 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1990656 - | + - + - > 5< + - + - | 0 - | 3598 - -> 00000080:00200000:3.0:1775764896.194144:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000f2f4eb2 00000080:00200000:3.0:1775764896.194145:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.194147:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.194155:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.194163:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.194170:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.194177:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.194188:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.194191:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.194193:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.194202:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.194205:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.194208:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.194211:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194219:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.194227:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.194229:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.194233:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.194236:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24078, 24078] 00000080:00200000:3.0:1775764896.194246:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.194250:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.194280:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98627584 i_blocks 90112 00000080:00200000:3.0:1775764896.194284:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.194299:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98627584, count: 4096 00000080:00200000:3.0:1775764896.194309:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98627584, bytes: 4096 00000080:00200000:3.0:1775764896.194311:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.194315:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24080, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.194317:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98627584, 98631680) 0 0 00000080:00200000:3.0:1775764896.194319:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.194324:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24079, 24079] 00020000:00200000:3.0:1775764896.194327:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98627584 chunk: [98627584, 98631680) 98631680, 4096 00020000:00200000:3.0:1775764896.194329:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.194340:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24079, 24079] 00000008:00010000:3.0:1775764896.194344:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.194347:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.194354:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194363:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98627584 98631679) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194370:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194384:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194391:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.194399:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98627584, 98631680) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.194404:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24079 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.194425:0:18302:0:(rw26.c:858:ll_write_end()) pos 98627584, len 4096, copied 4096 00000080:00200000:3.0:1775764896.194429:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003af2d47e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.194431:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cdae16e8 17ffffc0001001 3:0 ffff90c212820a80 24079 lru 00000080:00200000:3.0:1775764896.194436:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000091ed3955 24079: 1< 0 - - > 2< 98627584 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1986560 - | + - + - > 5< + - + - | 0 - | 3599 - -> 00000080:00200000:3.0:1775764896.194520:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003af2d47e 00000080:00200000:3.0:1775764896.194522:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.194524:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.194534:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.194542:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.194549:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.194556:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.194572:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.194575:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.194577:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.194586:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.194589:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.194592:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.194595:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194603:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.194609:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.194610:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.194613:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.194615:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24079, 24079] 00000080:00200000:3.0:1775764896.194623:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.194625:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.194644:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98631680 i_blocks 90112 00000080:00200000:3.0:1775764896.194647:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.194658:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98631680, count: 4096 00000080:00200000:3.0:1775764896.194664:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98631680, bytes: 4096 00000080:00200000:3.0:1775764896.194666:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.194668:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24081, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.194669:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98631680, 98635776) 0 0 00000080:00200000:3.0:1775764896.194671:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.194674:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24080, 24080] 00020000:00200000:3.0:1775764896.194677:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98631680 chunk: [98631680, 98635776) 98635776, 4096 00020000:00200000:3.0:1775764896.194678:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.194685:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24080, 24080] 00000008:00010000:3.0:1775764896.194688:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.194690:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.194695:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194701:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98631680 98635775) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194706:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194711:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194715:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.194720:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98631680, 98635776) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.194723:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24080 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.194736:0:18302:0:(rw26.c:858:ll_write_end()) pos 98631680, len 4096, copied 4096 00000080:00200000:3.0:1775764896.194739:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007e903b7e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.194741:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f5ac9f4f 17ffffc0001001 3:0 ffff90c212820700 24080 lru 00000080:00200000:3.0:1775764896.194744:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cc34c606 24080: 1< 0 - - > 2< 98631680 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1982464 - | + - + - > 5< + - + - | 0 - | 3600 - -> 00000080:00200000:3.0:1775764896.194746:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007e903b7e 00000080:00200000:3.0:1775764896.194747:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.194748:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.194753:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.194758:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.194763:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.194767:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.194774:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.194776:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.194778:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.194787:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.194790:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.194792:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.194795:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194801:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.194809:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.194811:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.194815:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.194817:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24080, 24080] 00000080:00200000:3.0:1775764896.194828:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.194832:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.194860:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98635776 i_blocks 90112 00000080:00200000:3.0:1775764896.194864:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.194904:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98635776, count: 4096 00000080:00200000:3.0:1775764896.194915:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98635776, bytes: 4096 00000080:00200000:3.0:1775764896.194917:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.194921:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24082, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.194923:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98635776, 98639872) 0 0 00000080:00200000:3.0:1775764896.194925:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.194931:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24081, 24081] 00020000:00200000:3.0:1775764896.194933:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98635776 chunk: [98635776, 98639872) 98639872, 4096 00020000:00200000:3.0:1775764896.194935:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.194946:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24081, 24081] 00000008:00010000:3.0:1775764896.194951:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.194953:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.194961:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194969:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98635776 98639871) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194977:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194985:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.194991:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.195000:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98635776, 98639872) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.195005:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24081 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.195022:0:18302:0:(rw26.c:858:ll_write_end()) pos 98635776, len 4096, copied 4096 00000080:00200000:3.0:1775764896.195027:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c4758a61[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.195030:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002f042525 17ffffc0001001 3:0 ffff90c212820c40 24081 lru 00000080:00200000:3.0:1775764896.195035:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000025825b71 24081: 1< 0 - - > 2< 98635776 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1978368 - | + - + - > 5< + - + - | 0 - | 3601 - -> 00000080:00200000:3.0:1775764896.195037:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c4758a61 00000080:00200000:3.0:1775764896.195039:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.195041:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.195048:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.195056:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.195063:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.195070:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.195081:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.195084:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.195086:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.195095:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.195098:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.195100:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.195107:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195123:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.195132:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.195133:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.195138:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.195141:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24081, 24081] 00000080:00200000:3.0:1775764896.195150:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.195153:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.195180:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98639872 i_blocks 90112 00000080:00200000:3.0:1775764896.195183:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.195198:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98639872, count: 4096 00000080:00200000:3.0:1775764896.195207:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98639872, bytes: 4096 00000080:00200000:3.0:1775764896.195209:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.195212:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24083, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.195214:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98639872, 98643968) 0 0 00000080:00200000:3.0:1775764896.195216:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.195221:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24082, 24082] 00020000:00200000:3.0:1775764896.195223:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98639872 chunk: [98639872, 98643968) 98643968, 4096 00020000:00200000:3.0:1775764896.195225:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.195235:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24082, 24082] 00000008:00010000:3.0:1775764896.195240:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.195242:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.195249:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195257:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98639872 98643967) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195265:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195272:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195279:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.195287:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98639872, 98643968) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.195292:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24082 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.195309:0:18302:0:(rw26.c:858:ll_write_end()) pos 98639872, len 4096, copied 4096 00000080:00200000:3.0:1775764896.195313:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009314c80a[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.195315:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000eba78270 17ffffc0001001 3:0 ffff90c2128208c0 24082 lru 00000080:00200000:3.0:1775764896.195320:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000721d3af8 24082: 1< 0 - - > 2< 98639872 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1974272 - | + - + - > 5< + - + - | 0 - | 3602 - -> 00000080:00200000:3.0:1775764896.195322:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009314c80a 00000080:00200000:3.0:1775764896.195323:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.195325:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.195332:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.195340:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.195346:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.195358:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.195370:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.195373:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.195375:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.195384:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.195386:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.195389:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.195393:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195401:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.195410:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.195412:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.195416:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.195420:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24082, 24082] 00000080:00200000:3.0:1775764896.195430:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.195433:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.195549:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98643968 i_blocks 90112 00000080:00200000:3.0:1775764896.195553:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.195569:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98643968, count: 4096 00000080:00200000:3.0:1775764896.195580:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98643968, bytes: 4096 00000080:00200000:3.0:1775764896.195583:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.195588:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24084, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.195589:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98643968, 98648064) 0 0 00000080:00200000:3.0:1775764896.195592:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.195597:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24083, 24083] 00020000:00200000:3.0:1775764896.195600:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98643968 chunk: [98643968, 98648064) 98648064, 4096 00020000:00200000:3.0:1775764896.195602:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.195614:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24083, 24083] 00000008:00010000:3.0:1775764896.195619:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.195621:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.195629:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195638:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98643968 98648063) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195646:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195654:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195661:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.195669:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98643968, 98648064) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.195674:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24083 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.195690:0:18302:0:(rw26.c:858:ll_write_end()) pos 98643968, len 4096, copied 4096 00000080:00200000:3.0:1775764896.195694:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009172769e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.195696:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f7bc7147 17ffffc0001001 3:0 ffff90c212820b60 24083 lru 00000080:00200000:3.0:1775764896.195701:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000695ece6e 24083: 1< 0 - - > 2< 98643968 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1970176 - | + - + - > 5< + - + - | 0 - | 3603 - -> 00000080:00200000:3.0:1775764896.195704:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009172769e 00000080:00200000:3.0:1775764896.195705:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.195707:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.195715:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.195723:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.195729:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.195737:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.195749:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.195752:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.195754:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.195765:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.195767:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.195769:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.195773:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195782:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.195790:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.195793:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.195797:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.195799:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24083, 24083] 00000080:00200000:3.0:1775764896.195809:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.195812:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.195841:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98648064 i_blocks 90112 00000080:00200000:3.0:1775764896.195844:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.195860:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98648064, count: 4096 00000080:00200000:3.0:1775764896.195896:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98648064, bytes: 4096 00000080:00200000:3.0:1775764896.195899:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.195903:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24085, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.195905:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98648064, 98652160) 0 0 00000080:00200000:3.0:1775764896.195907:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.195912:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24084, 24084] 00020000:00200000:3.0:1775764896.195916:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98648064 chunk: [98648064, 98652160) 98652160, 4096 00020000:00200000:3.0:1775764896.195918:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.195931:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24084, 24084] 00000008:00010000:3.0:1775764896.195936:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.195939:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.195947:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195956:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98648064 98652159) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195965:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195972:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.195985:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.195994:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98648064, 98652160) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.195999:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24084 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.196015:0:18302:0:(rw26.c:858:ll_write_end()) pos 98648064, len 4096, copied 4096 00000080:00200000:3.0:1775764896.196019:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000099e520f4[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.196022:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000000a306ac 17ffffc0001001 3:0 ffff90c212820000 24084 lru 00000080:00200000:3.0:1775764896.196027:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008a22c8fd 24084: 1< 0 - - > 2< 98648064 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1966080 - | + - + - > 5< + - + - | 0 - | 3604 - -> 00000080:00200000:3.0:1775764896.196029:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000099e520f4 00000080:00200000:3.0:1775764896.196030:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.196031:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.196039:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.196047:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.196054:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.196062:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.196073:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.196076:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.196078:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.196088:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.196091:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.196093:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.196097:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196106:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.196124:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.196126:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.196130:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.196133:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24084, 24084] 00000080:00200000:3.0:1775764896.196143:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.196146:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.196175:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98652160 i_blocks 90112 00000080:00200000:3.0:1775764896.196179:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.196196:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98652160, count: 4096 00000080:00200000:3.0:1775764896.196207:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98652160, bytes: 4096 00000080:00200000:3.0:1775764896.196209:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.196213:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24086, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.196215:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98652160, 98656256) 0 0 00000080:00200000:3.0:1775764896.196217:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.196222:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24085, 24085] 00020000:00200000:3.0:1775764896.196225:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98652160 chunk: [98652160, 98656256) 98656256, 4096 00020000:00200000:3.0:1775764896.196228:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.196240:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24085, 24085] 00000008:00010000:3.0:1775764896.196245:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.196248:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.196256:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196266:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98652160 98656255) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196275:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196282:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196289:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.196296:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98652160, 98656256) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.196302:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24085 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.196320:0:18302:0:(rw26.c:858:ll_write_end()) pos 98652160, len 4096, copied 4096 00000080:00200000:3.0:1775764896.196324:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f47558dd[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.196327:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000986630ce 17ffffc0001001 3:0 ffff90c212820ee0 24085 lru 00000080:00200000:3.0:1775764896.196333:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e7661429 24085: 1< 0 - - > 2< 98652160 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1961984 - | + - + - > 5< + - + - | 0 - | 3605 - -> 00000080:00200000:3.0:1775764896.196335:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f47558dd 00000080:00200000:3.0:1775764896.196336:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.196338:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.196346:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.196353:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.196360:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.196367:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.196379:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.196382:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.196384:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.196393:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.196395:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.196398:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.196402:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196410:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.196418:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.196421:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.196425:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.196428:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24085, 24085] 00000080:00200000:3.0:1775764896.196524:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.196528:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.196560:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98656256 i_blocks 90112 00000080:00200000:3.0:1775764896.196564:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.196582:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98656256, count: 4096 00000080:00200000:3.0:1775764896.196592:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98656256, bytes: 4096 00000080:00200000:3.0:1775764896.196595:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.196599:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24087, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.196600:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98656256, 98660352) 0 0 00000080:00200000:3.0:1775764896.196603:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.196608:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24086, 24086] 00020000:00200000:3.0:1775764896.196611:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98656256 chunk: [98656256, 98660352) 98660352, 4096 00020000:00200000:3.0:1775764896.196613:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.196626:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24086, 24086] 00000008:00010000:3.0:1775764896.196631:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.196634:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.196642:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196651:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98656256 98660351) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196660:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196668:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196676:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.196684:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98656256, 98660352) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.196689:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24086 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.196712:0:18302:0:(rw26.c:858:ll_write_end()) pos 98656256, len 4096, copied 4096 00000080:00200000:3.0:1775764896.196717:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009a305cf7[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.196720:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000072b7f4ab 17ffffc0001021 2:0 ffff90c2128200e0 24086 lru 00000080:00200000:3.0:1775764896.196726:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ad86c97a 24086: 1< 0 - - > 2< 98656256 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1957888 - | + - + - > 5< + - + - | 0 - | 3606 - -> 00000080:00200000:3.0:1775764896.196728:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009a305cf7 00000080:00200000:3.0:1775764896.196729:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.196731:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.196740:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.196748:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.196755:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.196763:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.196775:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.196777:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.196780:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.196789:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.196792:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.196794:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.196798:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196805:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.196814:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.196817:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.196821:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.196824:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24086, 24086] 00000080:00200000:3.0:1775764896.196835:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.196838:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.196867:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98660352 i_blocks 90112 00000080:00200000:3.0:1775764896.196892:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.196909:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98660352, count: 4096 00000080:00200000:3.0:1775764896.196921:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98660352, bytes: 4096 00000080:00200000:3.0:1775764896.196924:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.196927:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24088, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.196929:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98660352, 98664448) 0 0 00000080:00200000:3.0:1775764896.196931:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.196937:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24087, 24087] 00020000:00200000:3.0:1775764896.196940:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98660352 chunk: [98660352, 98664448) 98664448, 4096 00020000:00200000:3.0:1775764896.196942:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.196954:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24087, 24087] 00000008:00010000:3.0:1775764896.196959:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.196962:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.196969:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196978:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98660352 98664447) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.196987:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197000:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197007:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.197016:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98660352, 98664448) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.197022:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24087 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.197038:0:18302:0:(rw26.c:858:ll_write_end()) pos 98660352, len 4096, copied 4096 00000080:00200000:3.0:1775764896.197042:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000073d7d2fb[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.197045:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000045620a8a 17ffffc0001001 3:0 ffff90c2128201c0 24087 lru 00000080:00200000:3.0:1775764896.197050:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009be545b5 24087: 1< 0 - - > 2< 98660352 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1953792 - | + - + - > 5< + - + - | 0 - | 3607 - -> 00000080:00200000:3.0:1775764896.197052:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000073d7d2fb 00000080:00200000:3.0:1775764896.197053:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.197055:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.197063:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.197070:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.197076:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.197083:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.197099:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.197101:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.197103:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.197123:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.197126:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.197128:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.197132:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197139:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.197147:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.197149:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.197153:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.197156:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24087, 24087] 00000080:00200000:3.0:1775764896.197166:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.197169:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.197197:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98664448 i_blocks 90112 00000080:00200000:3.0:1775764896.197201:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.197217:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98664448, count: 4096 00000080:00200000:3.0:1775764896.197227:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98664448, bytes: 4096 00000080:00200000:3.0:1775764896.197230:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.197234:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24089, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.197235:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98664448, 98668544) 0 0 00000080:00200000:3.0:1775764896.197238:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.197243:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24088, 24088] 00020000:00200000:3.0:1775764896.197247:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98664448 chunk: [98664448, 98668544) 98668544, 4096 00020000:00200000:3.0:1775764896.197249:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.197260:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24088, 24088] 00000008:00010000:3.0:1775764896.197266:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.197268:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.197276:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197284:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98664448 98668543) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197292:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197299:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197305:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.197313:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98664448, 98668544) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.197318:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24088 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.197334:0:18302:0:(rw26.c:858:ll_write_end()) pos 98664448, len 4096, copied 4096 00000080:00200000:3.0:1775764896.197338:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b7799e3f[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.197341:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007091c36a 17ffffc0001001 3:0 ffff90c2128207e0 24088 lru 00000080:00200000:3.0:1775764896.197346:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007167b55b 24088: 1< 0 - - > 2< 98664448 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1949696 - | + - + - > 5< + - + - | 0 - | 3608 - -> 00000080:00200000:3.0:1775764896.197348:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b7799e3f 00000080:00200000:3.0:1775764896.197349:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.197351:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.197359:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.197366:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.197374:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.197382:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.197393:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.197396:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.197398:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.197408:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.197410:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.197413:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.197417:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197425:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.197433:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.197436:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.197521:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.197524:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24088, 24088] 00000080:00200000:3.0:1775764896.197536:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.197539:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.197571:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98668544 i_blocks 90112 00000080:00200000:3.0:1775764896.197575:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.197590:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98668544, count: 4096 00000080:00200000:3.0:1775764896.197599:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98668544, bytes: 4096 00000080:00200000:3.0:1775764896.197602:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.197605:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24090, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.197606:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98668544, 98672640) 0 0 00000080:00200000:3.0:1775764896.197608:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.197613:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24089, 24089] 00020000:00200000:3.0:1775764896.197616:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98668544 chunk: [98668544, 98672640) 98672640, 4096 00020000:00200000:3.0:1775764896.197618:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.197628:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24089, 24089] 00000008:00010000:3.0:1775764896.197632:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.197634:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.197642:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197652:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98668544 98672639) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197660:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197668:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197675:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.197683:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98668544, 98672640) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.197688:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24089 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.197713:0:18302:0:(rw26.c:858:ll_write_end()) pos 98668544, len 4096, copied 4096 00000080:00200000:3.0:1775764896.197717:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000657b316e[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.197720:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000004db2082 17ffffc0001001 3:0 ffff90c212821380 24089 lru 00000080:00200000:3.0:1775764896.197726:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003a052793 24089: 1< 0 - - > 2< 98668544 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1945600 - | + - + - > 5< + - + - | 0 - | 3609 - -> 00000080:00200000:3.0:1775764896.197728:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000657b316e 00000080:00200000:3.0:1775764896.197729:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.197731:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.197739:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.197747:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.197754:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.197761:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.197773:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.197775:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.197778:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.197787:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.197790:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.197792:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.197796:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197804:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.197811:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.197814:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.197818:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.197820:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24089, 24089] 00000080:00200000:3.0:1775764896.197831:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.197834:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.197863:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98672640 i_blocks 90112 00000080:00200000:3.0:1775764896.197867:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.197903:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98672640, count: 4096 00000080:00200000:3.0:1775764896.197914:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98672640, bytes: 4096 00000080:00200000:3.0:1775764896.197917:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.197920:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24091, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.197922:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98672640, 98676736) 0 0 00000080:00200000:3.0:1775764896.197924:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.197929:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24090, 24090] 00020000:00200000:3.0:1775764896.197932:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98672640 chunk: [98672640, 98676736) 98676736, 4096 00020000:00200000:3.0:1775764896.197935:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.197946:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24090, 24090] 00000008:00010000:3.0:1775764896.197951:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.197954:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.197962:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197970:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98672640 98676735) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197979:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197986:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.197993:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.198000:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98672640, 98676736) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.198005:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24090 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.198021:0:18302:0:(rw26.c:858:ll_write_end()) pos 98672640, len 4096, copied 4096 00000080:00200000:3.0:1775764896.198025:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008098cfc9[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.198028:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000248aeebf 17ffffc0001001 3:0 ffff90c212821620 24090 lru 00000080:00200000:3.0:1775764896.198034:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a59d11b2 24090: 1< 0 - - > 2< 98672640 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1941504 - | + - + - > 5< + - + - | 0 - | 3610 - -> 00000080:00200000:3.0:1775764896.198036:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008098cfc9 00000080:00200000:3.0:1775764896.198037:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.198039:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.198047:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.198056:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.198063:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.198076:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.198089:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.198091:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.198094:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.198104:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.198107:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.198109:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.198124:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198133:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.198142:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.198144:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.198149:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.198152:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24090, 24090] 00000080:00200000:3.0:1775764896.198164:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.198167:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.198198:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98676736 i_blocks 90112 00000080:00200000:3.0:1775764896.198202:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.198219:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98676736, count: 4096 00000080:00200000:3.0:1775764896.198230:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98676736, bytes: 4096 00000080:00200000:3.0:1775764896.198233:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.198237:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24092, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.198238:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98676736, 98680832) 0 0 00000080:00200000:3.0:1775764896.198240:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.198246:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24091, 24091] 00020000:00200000:3.0:1775764896.198249:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98676736 chunk: [98676736, 98680832) 98680832, 4096 00020000:00200000:3.0:1775764896.198251:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.198263:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24091, 24091] 00000008:00010000:3.0:1775764896.198268:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.198271:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.198279:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198287:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98676736 98680831) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198296:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198303:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198310:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.198319:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98676736, 98680832) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.198324:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24091 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.198341:0:18302:0:(rw26.c:858:ll_write_end()) pos 98676736, len 4096, copied 4096 00000080:00200000:3.0:1775764896.198345:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e6dcf855[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.198348:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003335d663 17ffffc0001001 3:0 ffff90c212821e00 24091 lru 00000080:00200000:3.0:1775764896.198353:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009b34e188 24091: 1< 0 - - > 2< 98676736 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1937408 - | + - + - > 5< + - + - | 0 - | 3611 - -> 00000080:00200000:3.0:1775764896.198355:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e6dcf855 00000080:00200000:3.0:1775764896.198356:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.198358:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.198366:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.198373:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.198381:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.198389:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.198400:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.198403:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.198405:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.198416:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.198418:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.198421:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.198425:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198433:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.198523:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.198526:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.198531:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.198534:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24091, 24091] 00000080:00200000:3.0:1775764896.198546:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.198549:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.198582:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98680832 i_blocks 90112 00000080:00200000:3.0:1775764896.198586:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.198603:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98680832, count: 4096 00000080:00200000:3.0:1775764896.198614:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98680832, bytes: 4096 00000080:00200000:3.0:1775764896.198617:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.198620:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24093, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.198622:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98680832, 98684928) 0 0 00000080:00200000:3.0:1775764896.198624:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.198629:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24092, 24092] 00020000:00200000:3.0:1775764896.198632:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98680832 chunk: [98680832, 98684928) 98684928, 4096 00020000:00200000:3.0:1775764896.198634:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.198645:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24092, 24092] 00000008:00010000:3.0:1775764896.198649:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.198652:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.198660:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198669:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98680832 98684927) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198677:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198685:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198697:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.198705:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98680832, 98684928) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.198709:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24092 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.198725:0:18302:0:(rw26.c:858:ll_write_end()) pos 98680832, len 4096, copied 4096 00000080:00200000:3.0:1775764896.198729:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000028b0a777[3 00000000b99c1f5d 2 1 000000009a32787f] 00000080:00200000:3.0:1775764896.198732:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006718c9da 17ffffc0001001 3:0 ffff90c212821540 24092 lru 00000080:00200000:3.0:1775764896.198738:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000240d8b6a 24092: 1< 0 - - > 2< 98680832 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1933312 - | + - + - > 5< + - + - | 0 - | 3612 - -> 00000080:00200000:3.0:1775764896.198740:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000028b0a777 00000080:00200000:3.0:1775764896.198741:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.198743:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.198750:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.198759:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009459d49a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.198766:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.198774:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.198785:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.198788:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.198791:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.198800:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.198803:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.198805:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.198808:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.198816:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.198824:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a6143f8 (ffff90c203cb9998/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.198826:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9998/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.198830:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.198833:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24092, 24092] 00000080:00200000:3.0:1775764896.198844:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.198847:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.211020:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98836480 i_blocks 90112 00000080:00200000:3.0:1775764896.211024:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.211046:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98836480, count: 4096 00000080:00200000:3.0:1775764896.211064:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98836480, bytes: 4096 00000080:00200000:3.0:1775764896.211068:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.211071:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24131, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.211073:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98836480, 98840576) 0 0 00000080:00200000:3.0:1775764896.211076:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.211082:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24130, 24130] 00020000:00200000:3.0:1775764896.211085:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98836480 chunk: [98836480, 98840576) 98840576, 4096 00020000:00200000:3.0:1775764896.211087:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.211103:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24130, 24130] 00000008:00010000:3.0:1775764896.211110:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.211125:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.211135:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211145:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98836480 98840575) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211154:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211162:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211169:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.211176:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98836480, 98840576) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.211182:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24130 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.211201:0:18302:0:(rw26.c:858:ll_write_end()) pos 98836480, len 4096, copied 4096 00000080:00200000:3.0:1775764896.211205:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000031fc1b47[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.211208:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000737cfc31 17ffffc0001001 3:0 ffff90c212821460 24130 lru 00000080:00200000:3.0:1775764896.211213:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b28e9028 24130: 1< 0 - - > 2< 98836480 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1777664 - | + - + - > 5< + - + - | 0 - | 3650 - -> 00000080:00200000:3.0:1775764896.211215:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000031fc1b47 00000080:00200000:3.0:1775764896.211216:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.211218:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.211226:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.211234:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.211242:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.211250:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.211261:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.211263:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.211266:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.211272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.211276:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.211278:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.211282:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211290:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.211298:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.211300:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.211305:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.211307:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24130, 24130] 00000080:00200000:3.0:1775764896.211323:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.211326:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.211331:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98840576 i_blocks 90112 00000080:00200000:3.0:1775764896.211334:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.211352:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98840576, count: 4096 00000080:00200000:3.0:1775764896.211363:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98840576, bytes: 4096 00000080:00200000:3.0:1775764896.211366:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.211370:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24132, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.211371:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98840576, 98844672) 0 0 00000080:00200000:3.0:1775764896.211374:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.211379:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24131, 24131] 00020000:00200000:3.0:1775764896.211382:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98840576 chunk: [98840576, 98844672) 98844672, 4096 00020000:00200000:3.0:1775764896.211384:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.211395:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24131, 24131] 00000008:00010000:3.0:1775764896.211399:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.211402:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.211409:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211418:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98840576 98844671) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211427:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211434:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211596:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.211606:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98840576, 98844672) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.211613:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24131 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.211633:0:18302:0:(rw26.c:858:ll_write_end()) pos 98840576, len 4096, copied 4096 00000080:00200000:3.0:1775764896.211638:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000386ea090[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.211641:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fce0175f 17ffffc0001001 3:0 ffff90c212821a80 24131 lru 00000080:00200000:3.0:1775764896.211647:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000409012d5 24131: 1< 0 - - > 2< 98840576 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1773568 - | + - + - > 5< + - + - | 0 - | 3651 - -> 00000080:00200000:3.0:1775764896.211649:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000386ea090 00000080:00200000:3.0:1775764896.211650:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.211653:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.211661:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.211669:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.211677:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.211684:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.211695:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.211698:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.211701:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.211714:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.211717:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.211719:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.211724:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211732:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.211740:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.211743:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.211747:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.211750:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24131, 24131] 00000080:00200000:3.0:1775764896.211761:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.211764:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.211795:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98844672 i_blocks 90112 00000080:00200000:3.0:1775764896.211799:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.211814:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98844672, count: 4096 00000080:00200000:3.0:1775764896.211825:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98844672, bytes: 4096 00000080:00200000:3.0:1775764896.211828:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.211831:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24133, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.211832:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98844672, 98848768) 0 0 00000080:00200000:3.0:1775764896.211835:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.211840:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24132, 24132] 00020000:00200000:3.0:1775764896.211843:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98844672 chunk: [98844672, 98848768) 98848768, 4096 00020000:00200000:3.0:1775764896.211845:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.211868:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24132, 24132] 00000008:00010000:3.0:1775764896.211896:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.211899:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.211907:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211915:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98844672 98848767) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211923:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211935:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.211942:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.211950:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98844672, 98848768) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.211955:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24132 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.211971:0:18302:0:(rw26.c:858:ll_write_end()) pos 98844672, len 4096, copied 4096 00000080:00200000:3.0:1775764896.211975:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002b93dec7[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.211978:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000036de125c 17ffffc0001001 3:0 ffff90c212821700 24132 lru 00000080:00200000:3.0:1775764896.211984:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000080054a8e 24132: 1< 0 - - > 2< 98844672 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1769472 - | + - + - > 5< + - + - | 0 - | 3652 - -> 00000080:00200000:3.0:1775764896.211986:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002b93dec7 00000080:00200000:3.0:1775764896.211987:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.211989:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.211997:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.212005:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.212012:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.212020:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.212037:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.212040:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.212042:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.212052:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.212054:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.212056:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.212060:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212067:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.212075:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.212077:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.212081:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.212084:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24132, 24132] 00000080:00200000:3.0:1775764896.212094:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.212097:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.212140:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98848768 i_blocks 90112 00000080:00200000:3.0:1775764896.212144:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.212160:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98848768, count: 4096 00000080:00200000:3.0:1775764896.212170:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98848768, bytes: 4096 00000080:00200000:3.0:1775764896.212173:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.212176:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24134, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.212177:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98848768, 98852864) 0 0 00000080:00200000:3.0:1775764896.212180:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.212185:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24133, 24133] 00020000:00200000:3.0:1775764896.212188:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98848768 chunk: [98848768, 98852864) 98852864, 4096 00020000:00200000:3.0:1775764896.212191:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.212202:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24133, 24133] 00000008:00010000:3.0:1775764896.212208:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.212210:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.212218:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212226:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98848768 98852863) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212233:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212240:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212247:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.212254:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98848768, 98852864) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.212259:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24133 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.212276:0:18302:0:(rw26.c:858:ll_write_end()) pos 98848768, len 4096, copied 4096 00000080:00200000:3.0:1775764896.212280:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d1969e3e[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.212283:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000007fdd126 17ffffc0001001 3:0 ffff90c212821c40 24133 lru 00000080:00200000:3.0:1775764896.212289:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000087301e7d 24133: 1< 0 - - > 2< 98848768 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1765376 - | + - + - > 5< + - + - | 0 - | 3653 - -> 00000080:00200000:3.0:1775764896.212291:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d1969e3e 00000080:00200000:3.0:1775764896.212292:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.212294:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.212303:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.212310:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.212317:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.212326:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.212337:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.212340:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.212342:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.212352:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.212354:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.212357:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.212361:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212369:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.212377:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.212380:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.212384:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.212387:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24133, 24133] 00000080:00200000:3.0:1775764896.212398:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.212401:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.212428:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98852864 i_blocks 90112 00000080:00200000:3.0:1775764896.212431:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.212528:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98852864, count: 4096 00000080:00200000:3.0:1775764896.212540:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98852864, bytes: 4096 00000080:00200000:3.0:1775764896.212543:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.212547:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24135, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.212549:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98852864, 98856960) 0 0 00000080:00200000:3.0:1775764896.212551:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.212556:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24134, 24134] 00020000:00200000:3.0:1775764896.212559:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98852864 chunk: [98852864, 98856960) 98856960, 4096 00020000:00200000:3.0:1775764896.212562:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.212573:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24134, 24134] 00000008:00010000:3.0:1775764896.212579:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.212581:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.212589:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212598:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98852864 98856959) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212607:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212614:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212622:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.212630:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98852864, 98856960) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.212636:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24134 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.212652:0:18302:0:(rw26.c:858:ll_write_end()) pos 98852864, len 4096, copied 4096 00000080:00200000:3.0:1775764896.212657:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003b5b5a60[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.212660:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b8809128 17ffffc0001001 3:0 ffff90c2128218c0 24134 lru 00000080:00200000:3.0:1775764896.212666:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007805248f 24134: 1< 0 - - > 2< 98852864 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1761280 - | + - + - > 5< + - + - | 0 - | 3654 - -> 00000080:00200000:3.0:1775764896.212668:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003b5b5a60 00000080:00200000:3.0:1775764896.212669:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.212671:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.212679:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.212687:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.212695:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.212703:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.212715:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.212718:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.212720:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.212730:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.212733:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.212736:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.212740:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212749:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.212757:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.212760:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.212764:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.212767:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24134, 24134] 00000080:00200000:3.0:1775764896.212777:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.212781:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.212810:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98856960 i_blocks 90112 00000080:00200000:3.0:1775764896.212814:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.212830:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98856960, count: 4096 00000080:00200000:3.0:1775764896.212840:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98856960, bytes: 4096 00000080:00200000:3.0:1775764896.212843:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.212846:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24136, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.212848:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98856960, 98861056) 0 0 00000080:00200000:3.0:1775764896.212850:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.212856:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24135, 24135] 00020000:00200000:3.0:1775764896.212859:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98856960 chunk: [98856960, 98861056) 98861056, 4096 00020000:00200000:3.0:1775764896.212862:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.212897:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24135, 24135] 00000008:00010000:3.0:1775764896.212903:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.212905:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.212912:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212920:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98856960 98861055) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212929:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212935:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.212942:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.212949:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98856960, 98861056) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.212955:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24135 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.212971:0:18302:0:(rw26.c:858:ll_write_end()) pos 98856960, len 4096, copied 4096 00000080:00200000:3.0:1775764896.212976:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fba9be3a[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.212978:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fc9bb6cf 17ffffc0001001 3:0 ffff90c212821b60 24135 lru 00000080:00200000:3.0:1775764896.212984:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e549f7f1 24135: 1< 0 - - > 2< 98856960 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1757184 - | + - + - > 5< + - + - | 0 - | 3655 - -> 00000080:00200000:3.0:1775764896.212986:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fba9be3a 00000080:00200000:3.0:1775764896.212987:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.212989:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.212997:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.213005:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.213011:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.213023:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.213035:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.213038:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.213040:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.213049:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.213052:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.213055:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.213058:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213066:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.213073:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.213075:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.213079:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.213081:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24135, 24135] 00000080:00200000:3.0:1775764896.213091:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.213094:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.213136:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98861056 i_blocks 90112 00000080:00200000:3.0:1775764896.213140:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.213156:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98861056, count: 4096 00000080:00200000:3.0:1775764896.213166:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98861056, bytes: 4096 00000080:00200000:3.0:1775764896.213168:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.213172:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24137, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.213174:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98861056, 98865152) 0 0 00000080:00200000:3.0:1775764896.213176:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.213181:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24136, 24136] 00020000:00200000:3.0:1775764896.213183:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98861056 chunk: [98861056, 98865152) 98865152, 4096 00020000:00200000:3.0:1775764896.213186:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.213196:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24136, 24136] 00000008:00010000:3.0:1775764896.213201:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.213204:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.213211:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213219:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98861056 98865151) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213227:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213234:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213241:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.213248:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98861056, 98865152) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.213253:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24136 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.213269:0:18302:0:(rw26.c:858:ll_write_end()) pos 98861056, len 4096, copied 4096 00000080:00200000:3.0:1775764896.213273:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000537b0057[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.213276:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000086ad23df 17ffffc0001001 3:0 ffff90c212821000 24136 lru 00000080:00200000:3.0:1775764896.213281:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000734798f9 24136: 1< 0 - - > 2< 98861056 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1753088 - | + - + - > 5< + - + - | 0 - | 3656 - -> 00000080:00200000:3.0:1775764896.213283:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000537b0057 00000080:00200000:3.0:1775764896.213284:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.213287:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.213293:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.213301:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.213307:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.213313:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.213323:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.213325:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.213328:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.213338:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.213340:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.213343:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.213346:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213354:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.213362:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.213365:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.213369:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.213371:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24136, 24136] 00000080:00200000:3.0:1775764896.213382:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.213385:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.213414:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98865152 i_blocks 90112 00000080:00200000:3.0:1775764896.213418:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.213434:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98865152, count: 4096 00000080:00200000:3.0:1775764896.213529:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98865152, bytes: 4096 00000080:00200000:3.0:1775764896.213533:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.213536:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24138, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.213538:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98865152, 98869248) 0 0 00000080:00200000:3.0:1775764896.213540:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.213546:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24137, 24137] 00020000:00200000:3.0:1775764896.213549:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98865152 chunk: [98865152, 98869248) 98869248, 4096 00020000:00200000:3.0:1775764896.213552:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.213563:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24137, 24137] 00000008:00010000:3.0:1775764896.213568:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.213571:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.213581:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213590:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98865152 98869247) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213599:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213606:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213619:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.213627:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98865152, 98869248) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.213633:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24137 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.213651:0:18302:0:(rw26.c:858:ll_write_end()) pos 98865152, len 4096, copied 4096 00000080:00200000:3.0:1775764896.213655:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000013f91cf6[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.213658:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cb29c9db 17ffffc0001001 3:0 ffff90c212821ee0 24137 lru 00000080:00200000:3.0:1775764896.213663:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b335164f 24137: 1< 0 - - > 2< 98865152 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1748992 - | + - + - > 5< + - + - | 0 - | 3657 - -> 00000080:00200000:3.0:1775764896.213665:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000013f91cf6 00000080:00200000:3.0:1775764896.213666:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.213668:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.213677:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.213684:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.213691:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.213697:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.213708:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.213728:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.213732:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.213742:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.213744:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.213747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.213750:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213757:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.213765:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.213768:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.213771:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.213774:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24137, 24137] 00000080:00200000:3.0:1775764896.213785:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.213788:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.213817:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98869248 i_blocks 90112 00000080:00200000:3.0:1775764896.213821:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.213836:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98869248, count: 4096 00000080:00200000:3.0:1775764896.213846:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98869248, bytes: 4096 00000080:00200000:3.0:1775764896.213851:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.213854:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24139, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.213856:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98869248, 98873344) 0 0 00000080:00200000:3.0:1775764896.213858:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.213863:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24138, 24138] 00020000:00200000:3.0:1775764896.213866:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98869248 chunk: [98869248, 98873344) 98873344, 4096 00020000:00200000:3.0:1775764896.213891:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.213904:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24138, 24138] 00000008:00010000:3.0:1775764896.213910:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.213913:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.213922:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213932:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98869248 98873343) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213939:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213946:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.213953:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.213961:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98869248, 98873344) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.213966:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24138 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.213990:0:18302:0:(rw26.c:858:ll_write_end()) pos 98869248, len 4096, copied 4096 00000080:00200000:3.0:1775764896.213994:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000016d553c9[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.213997:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b9157618 17ffffc0001021 2:0 ffff90c2128210e0 24138 lru 00000080:00200000:3.0:1775764896.214003:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003c385b07 24138: 1< 0 - - > 2< 98869248 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1744896 - | + - + - > 5< + - + - | 0 - | 3658 - -> 00000080:00200000:3.0:1775764896.214005:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000016d553c9 00000080:00200000:3.0:1775764896.214007:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.214009:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.214017:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.214025:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.214032:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.214039:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.214050:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.214053:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.214055:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.214064:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.214067:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.214069:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.214073:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214080:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.214088:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.214090:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.214093:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.214096:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24138, 24138] 00000080:00200000:3.0:1775764896.214106:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.214109:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.214151:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98873344 i_blocks 90112 00000080:00200000:3.0:1775764896.214155:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.214173:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98873344, count: 4096 00000080:00200000:3.0:1775764896.214184:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98873344, bytes: 4096 00000080:00200000:3.0:1775764896.214186:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.214190:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24140, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.214192:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98873344, 98877440) 0 0 00000080:00200000:3.0:1775764896.214194:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.214199:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24139, 24139] 00020000:00200000:3.0:1775764896.214202:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98873344 chunk: [98873344, 98877440) 98877440, 4096 00020000:00200000:3.0:1775764896.214204:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.214215:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24139, 24139] 00000008:00010000:3.0:1775764896.214221:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.214223:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.214231:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214240:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98873344 98877439) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214248:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214255:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214262:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.214270:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98873344, 98877440) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.214275:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24139 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.214291:0:18302:0:(rw26.c:858:ll_write_end()) pos 98873344, len 4096, copied 4096 00000080:00200000:3.0:1775764896.214296:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cd74c43b[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.214299:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000076ddeb4a 17ffffc0001001 3:0 ffff90c2128211c0 24139 lru 00000080:00200000:3.0:1775764896.214304:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000586197ee 24139: 1< 0 - - > 2< 98873344 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1740800 - | + - + - > 5< + - + - | 0 - | 3659 - -> 00000080:00200000:3.0:1775764896.214306:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cd74c43b 00000080:00200000:3.0:1775764896.214306:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.214308:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.214316:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.214323:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.214330:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.214338:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.214349:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.214352:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.214354:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.214364:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.214366:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.214368:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.214372:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214380:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.214388:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.214390:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.214394:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.214397:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24139, 24139] 00000080:00200000:3.0:1775764896.214407:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.214410:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.214520:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98877440 i_blocks 90112 00000080:00200000:3.0:1775764896.214525:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.214543:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98877440, count: 4096 00000080:00200000:3.0:1775764896.214555:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98877440, bytes: 4096 00000080:00200000:3.0:1775764896.214557:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.214561:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24141, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.214562:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98877440, 98881536) 0 0 00000080:00200000:3.0:1775764896.214564:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.214570:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24140, 24140] 00020000:00200000:3.0:1775764896.214573:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98877440 chunk: [98877440, 98881536) 98881536, 4096 00020000:00200000:3.0:1775764896.214575:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.214587:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24140, 24140] 00000008:00010000:3.0:1775764896.214592:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.214595:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.214603:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214612:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98877440 98881535) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214621:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214634:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214640:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.214649:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98877440, 98881536) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.214654:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24140 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.214676:0:18302:0:(rw26.c:858:ll_write_end()) pos 98877440, len 4096, copied 4096 00000080:00200000:3.0:1775764896.214680:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000098c223f4[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.214684:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000d2cb1f9 17ffffc0001001 3:0 ffff90c2128217e0 24140 lru 00000080:00200000:3.0:1775764896.214690:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000af14bbea 24140: 1< 0 - - > 2< 98877440 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1736704 - | + - + - > 5< + - + - | 0 - | 3660 - -> 00000080:00200000:3.0:1775764896.214692:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000098c223f4 00000080:00200000:3.0:1775764896.214693:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.214695:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.214703:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.214711:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.214717:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.214725:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.214741:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.214744:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.214747:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.214756:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.214758:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.214761:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.214764:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214772:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.214781:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.214784:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.214788:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.214790:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24140, 24140] 00000080:00200000:3.0:1775764896.214800:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.214803:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.214830:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98881536 i_blocks 90112 00000080:00200000:3.0:1775764896.214834:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.214850:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98881536, count: 4096 00000080:00200000:3.0:1775764896.214861:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98881536, bytes: 4096 00000080:00200000:3.0:1775764896.214864:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.214868:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24142, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.214892:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98881536, 98885632) 0 0 00000080:00200000:3.0:1775764896.214895:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.214900:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24141, 24141] 00020000:00200000:3.0:1775764896.214904:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98881536 chunk: [98881536, 98885632) 98885632, 4096 00020000:00200000:3.0:1775764896.214907:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.214920:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24141, 24141] 00000008:00010000:3.0:1775764896.214925:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.214928:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.214935:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214944:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98881536 98885631) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214952:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214960:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.214967:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.214974:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98881536, 98885632) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.214981:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24141 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.214997:0:18302:0:(rw26.c:858:ll_write_end()) pos 98881536, len 4096, copied 4096 00000080:00200000:3.0:1775764896.215001:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003ae7c8a4[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.215003:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009afaee0c 17ffffc0001001 3:0 ffff90c2128219a0 24141 lru 00000080:00200000:3.0:1775764896.215008:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000067a86961 24141: 1< 0 - - > 2< 98881536 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1732608 - | + - + - > 5< + - + - | 0 - | 3661 - -> 00000080:00200000:3.0:1775764896.215010:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003ae7c8a4 00000080:00200000:3.0:1775764896.215011:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.215013:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.215021:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.215028:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.215034:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.215041:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.215051:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.215054:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.215056:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.215065:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.215067:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.215069:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.215073:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215080:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.215087:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.215089:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.215094:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.215096:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24141, 24141] 00000080:00200000:3.0:1775764896.215107:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.215110:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.215151:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98885632 i_blocks 90112 00000080:00200000:3.0:1775764896.215155:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.215171:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98885632, count: 4096 00000080:00200000:3.0:1775764896.215181:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98885632, bytes: 4096 00000080:00200000:3.0:1775764896.215184:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.215188:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24143, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.215190:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98885632, 98889728) 0 0 00000080:00200000:3.0:1775764896.215192:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.215198:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24142, 24142] 00020000:00200000:3.0:1775764896.215200:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98885632 chunk: [98885632, 98889728) 98889728, 4096 00020000:00200000:3.0:1775764896.215203:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.215215:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24142, 24142] 00000008:00010000:3.0:1775764896.215220:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.215222:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.215230:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215238:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98885632 98889727) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215246:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215253:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215260:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.215268:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98885632, 98889728) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.215273:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24142 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.215289:0:18302:0:(rw26.c:858:ll_write_end()) pos 98885632, len 4096, copied 4096 00000080:00200000:3.0:1775764896.215293:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d49b0638[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.215296:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c8c00e88 17ffffc0001001 3:0 ffff90c2128212a0 24142 lru 00000080:00200000:3.0:1775764896.215301:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009e582ea2 24142: 1< 0 - - > 2< 98885632 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1728512 - | + - + - > 5< + - + - | 0 - | 3662 - -> 00000080:00200000:3.0:1775764896.215303:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d49b0638 00000080:00200000:3.0:1775764896.215304:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.215306:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.215313:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.215320:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.215327:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.215334:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.215346:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.215349:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.215351:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.215361:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.215364:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.215366:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.215371:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215379:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.215387:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.215389:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.215393:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.215396:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24142, 24142] 00000080:00200000:3.0:1775764896.215407:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.215410:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.215521:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98889728 i_blocks 90112 00000080:00200000:3.0:1775764896.215526:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.215544:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98889728, count: 4096 00000080:00200000:3.0:1775764896.215556:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98889728, bytes: 4096 00000080:00200000:3.0:1775764896.215559:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.215563:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24144, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.215564:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98889728, 98893824) 0 0 00000080:00200000:3.0:1775764896.215567:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.215573:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24143, 24143] 00020000:00200000:3.0:1775764896.215576:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98889728 chunk: [98889728, 98893824) 98893824, 4096 00020000:00200000:3.0:1775764896.215578:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.215588:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24143, 24143] 00000008:00010000:3.0:1775764896.215594:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.215596:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.215603:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215612:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98889728 98893823) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215621:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215628:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215635:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.215642:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98889728, 98893824) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.215647:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24143 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.215663:0:18302:0:(rw26.c:858:ll_write_end()) pos 98889728, len 4096, copied 4096 00000080:00200000:3.0:1775764896.215667:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006faca8b3[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.215670:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000016f6325 17ffffc0001001 3:0 ffff90c212821d20 24143 lru 00000080:00200000:3.0:1775764896.215675:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005ec97f91 24143: 1< 0 - - > 2< 98889728 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1724416 - | + - + - > 5< + - + - | 0 - | 3663 - -> 00000080:00200000:3.0:1775764896.215678:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006faca8b3 00000080:00200000:3.0:1775764896.215679:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.215681:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.215687:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.215695:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.215702:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.215715:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.215726:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.215729:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.215731:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.215742:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.215744:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.215747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.215751:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215759:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.215767:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.215769:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.215774:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.215776:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24143, 24143] 00000080:00200000:3.0:1775764896.215787:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.215790:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.215818:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98893824 i_blocks 90112 00000080:00200000:3.0:1775764896.215822:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.215838:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98893824, count: 4096 00000080:00200000:3.0:1775764896.215849:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98893824, bytes: 4096 00000080:00200000:3.0:1775764896.215852:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.215856:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24145, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.215857:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98893824, 98897920) 0 0 00000080:00200000:3.0:1775764896.215860:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.215865:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24144, 24144] 00020000:00200000:3.0:1775764896.215868:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98893824 chunk: [98893824, 98897920) 98897920, 4096 00020000:00200000:3.0:1775764896.215892:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.215905:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24144, 24144] 00000008:00010000:3.0:1775764896.215911:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.215913:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.215922:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215931:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98893824 98897919) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215940:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215947:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.215954:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.215962:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98893824, 98897920) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.215967:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24144 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.215994:0:18302:0:(rw26.c:858:ll_write_end()) pos 98893824, len 4096, copied 4096 00000080:00200000:3.0:1775764896.215999:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000beb8ae9c[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.216001:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d62ea348 17ffffc0001001 3:0 ffff90c2128257e0 24144 lru 00000080:00200000:3.0:1775764896.216007:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000007f97f3a 24144: 1< 0 - - > 2< 98893824 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1720320 - | + - + - > 5< + - + - | 0 - | 3664 - -> 00000080:00200000:3.0:1775764896.216009:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000beb8ae9c 00000080:00200000:3.0:1775764896.216010:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.216013:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.216021:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.216028:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.216035:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.216041:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.216051:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.216054:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.216057:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.216066:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.216068:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.216071:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.216074:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216081:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.216089:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.216092:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.216096:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.216099:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24144, 24144] 00000080:00200000:3.0:1775764896.216108:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.216112:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.216156:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98897920 i_blocks 90112 00000080:00200000:3.0:1775764896.216159:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.216175:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98897920, count: 4096 00000080:00200000:3.0:1775764896.216186:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98897920, bytes: 4096 00000080:00200000:3.0:1775764896.216189:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.216192:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24146, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.216194:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98897920, 98902016) 0 0 00000080:00200000:3.0:1775764896.216196:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.216201:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24145, 24145] 00020000:00200000:3.0:1775764896.216203:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98897920 chunk: [98897920, 98902016) 98902016, 4096 00020000:00200000:3.0:1775764896.216206:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.216217:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24145, 24145] 00000008:00010000:3.0:1775764896.216222:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.216225:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.216233:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216242:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98897920 98902015) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216251:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216259:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216271:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.216279:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98897920, 98902016) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.216285:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24145 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.216301:0:18302:0:(rw26.c:858:ll_write_end()) pos 98897920, len 4096, copied 4096 00000080:00200000:3.0:1775764896.216305:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b4d9c412[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.216309:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000035182ad0 17ffffc0001001 3:0 ffff90c2128259a0 24145 lru 00000080:00200000:3.0:1775764896.216314:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000000e3c82f 24145: 1< 0 - - > 2< 98897920 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1716224 - | + - + - > 5< + - + - | 0 - | 3665 - -> 00000080:00200000:3.0:1775764896.216316:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b4d9c412 00000080:00200000:3.0:1775764896.216318:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.216320:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.216328:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.216336:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.216343:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.216351:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.216362:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.216365:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.216368:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.216378:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.216380:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.216383:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.216386:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216394:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.216402:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.216404:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.216408:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.216411:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24145, 24145] 00000080:00200000:3.0:1775764896.216422:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.216425:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.216544:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98902016 i_blocks 90112 00000080:00200000:3.0:1775764896.216549:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.216566:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98902016, count: 4096 00000080:00200000:3.0:1775764896.216577:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98902016, bytes: 4096 00000080:00200000:3.0:1775764896.216580:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.216584:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24147, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.216585:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98902016, 98906112) 0 0 00000080:00200000:3.0:1775764896.216587:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.216593:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24146, 24146] 00020000:00200000:3.0:1775764896.216596:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98902016 chunk: [98902016, 98906112) 98906112, 4096 00020000:00200000:3.0:1775764896.216599:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.216611:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24146, 24146] 00000008:00010000:3.0:1775764896.216616:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.216619:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.216627:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216636:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98902016 98906111) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216644:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216651:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216658:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.216666:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98902016, 98906112) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.216672:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24146 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.216689:0:18302:0:(rw26.c:858:ll_write_end()) pos 98902016, len 4096, copied 4096 00000080:00200000:3.0:1775764896.216694:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000566c8de6[3 00000000b99c1f5d 2 1 00000000581222d7] 00000080:00200000:3.0:1775764896.216696:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000bf6f26f4 17ffffc0001001 3:0 ffff90c2128252a0 24146 lru 00000080:00200000:3.0:1775764896.216701:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000551d4138 24146: 1< 0 - - > 2< 98902016 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1712128 - | + - + - > 5< + - + - | 0 - | 3666 - -> 00000080:00200000:3.0:1775764896.216704:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000566c8de6 00000080:00200000:3.0:1775764896.216705:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.216707:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.216715:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.216723:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000a02cbbc0 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.216730:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.216737:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.216748:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.216751:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.216753:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.216763:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.216766:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.216768:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.216772:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.216780:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.216789:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614e48 (ffff90c203cb92d8/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.216791:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb92d8/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.216795:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.216797:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24146, 24146] 00000080:00200000:3.0:1775764896.216808:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.216811:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.217247:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98906112 i_blocks 90112 00000080:00200000:0.0:1775764896.217251:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.217273:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98906112, count: 4096 00000080:00200000:0.0:1775764896.217291:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98906112, bytes: 4096 00000080:00200000:0.0:1775764896.217295:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.217299:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24148, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.217300:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98906112, 98910208) 0 0 00000080:00200000:0.0:1775764896.217303:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.217310:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24147, 24147] 00020000:00200000:0.0:1775764896.217315:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98906112 chunk: [98906112, 98910208) 98910208, 4096 00020000:00200000:0.0:1775764896.217317:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.217335:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24147, 24147] 00000008:00010000:0.0:1775764896.217342:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.217346:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.217356:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217365:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98906112 98910207) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217374:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217382:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217389:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.217397:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98906112, 98910208) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.217403:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24147 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.217423:0:18302:0:(rw26.c:858:ll_write_end()) pos 98906112, len 4096, copied 4096 00000080:00200000:0.0:1775764896.217428:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ce349a1f[3 00000000b99c1f5d 2 1 000000003a48b37d] 00000080:00200000:0.0:1775764896.217431:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000037dd2f3a 17ffffc0001001 3:0 ffff90c212812c40 24147 lru 00000080:00200000:0.0:1775764896.217523:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001ef7aeaf 24147: 1< 0 - - > 2< 98906112 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1708032 - | + - + - > 5< + - + - | 0 - | 3667 - -> 00000080:00200000:0.0:1775764896.217527:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ce349a1f 00000080:00200000:0.0:1775764896.217528:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.217531:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.217543:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.217552:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.217559:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.217571:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.217583:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.217587:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.217589:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.217602:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.217605:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.217607:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.217611:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217619:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.217629:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.217632:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.217638:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.217641:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24147, 24147] 00000080:00200000:0.0:1775764896.217657:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.217661:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.217694:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98910208 i_blocks 90112 00000080:00200000:0.0:1775764896.217698:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.217717:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98910208, count: 4096 00000080:00200000:0.0:1775764896.217728:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98910208, bytes: 4096 00000080:00200000:0.0:1775764896.217731:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.217735:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24149, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.217737:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98910208, 98914304) 0 0 00000080:00200000:0.0:1775764896.217740:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.217745:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24148, 24148] 00020000:00200000:0.0:1775764896.217748:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98910208 chunk: [98910208, 98914304) 98914304, 4096 00020000:00200000:0.0:1775764896.217751:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.217763:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24148, 24148] 00000008:00010000:0.0:1775764896.217768:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.217770:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.217778:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217787:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98910208 98914303) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217794:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217801:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217807:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.217815:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98910208, 98914304) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.217821:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24148 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.217839:0:18302:0:(rw26.c:858:ll_write_end()) pos 98910208, len 4096, copied 4096 00000080:00200000:0.0:1775764896.217843:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000016532e86[3 00000000b99c1f5d 2 1 000000003a48b37d] 00000080:00200000:0.0:1775764896.217846:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004a914019 17ffffc0001001 3:0 ffff90c2128128c0 24148 lru 00000080:00200000:0.0:1775764896.217852:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a954028a 24148: 1< 0 - - > 2< 98910208 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1703936 - | + - + - > 5< + - + - | 0 - | 3668 - -> 00000080:00200000:0.0:1775764896.217854:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000016532e86 00000080:00200000:0.0:1775764896.217856:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.217858:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.217866:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.217903:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.217910:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.217918:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.217929:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.217933:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.217935:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.217943:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.217945:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.217948:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.217951:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.217960:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.217968:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.217971:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.217975:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.217978:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24148, 24148] 00000080:00200000:0.0:1775764896.217988:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.217991:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.218034:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98914304 i_blocks 90112 00000080:00200000:0.0:1775764896.218038:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.218053:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98914304, count: 4096 00000080:00200000:0.0:1775764896.218064:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98914304, bytes: 4096 00000080:00200000:0.0:1775764896.218066:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.218070:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24150, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.218071:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98914304, 98918400) 0 0 00000080:00200000:0.0:1775764896.218074:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.218079:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24149, 24149] 00020000:00200000:0.0:1775764896.218083:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98914304 chunk: [98914304, 98918400) 98918400, 4096 00020000:00200000:0.0:1775764896.218085:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.218096:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24149, 24149] 00000008:00010000:0.0:1775764896.218102:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.218104:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.218112:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218132:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98914304 98918399) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218139:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218146:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218156:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.218164:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98914304, 98918400) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.218169:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24149 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.218186:0:18302:0:(rw26.c:858:ll_write_end()) pos 98914304, len 4096, copied 4096 00000080:00200000:0.0:1775764896.218190:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000027ae0002[3 00000000b99c1f5d 2 1 000000003a48b37d] 00000080:00200000:0.0:1775764896.218193:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000080b7e39 17ffffc0001001 3:0 ffff90c212812b60 24149 lru 00000080:00200000:0.0:1775764896.218198:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000056a8f291 24149: 1< 0 - - > 2< 98914304 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1699840 - | + - + - > 5< + - + - | 0 - | 3669 - -> 00000080:00200000:0.0:1775764896.218200:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000027ae0002 00000080:00200000:0.0:1775764896.218201:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.218203:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.218211:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.218219:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.218225:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.218232:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.218243:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.218246:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.218249:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.218259:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.218262:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.218265:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.218269:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218278:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.218286:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.218289:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.218293:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.218296:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24149, 24149] 00000080:00200000:0.0:1775764896.218307:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.218310:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.218339:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98918400 i_blocks 90112 00000080:00200000:0.0:1775764896.218343:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.218359:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98918400, count: 4096 00000080:00200000:0.0:1775764896.218370:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98918400, bytes: 4096 00000080:00200000:0.0:1775764896.218372:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.218376:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24151, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.218377:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98918400, 98922496) 0 0 00000080:00200000:0.0:1775764896.218380:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.218384:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24150, 24150] 00020000:00200000:0.0:1775764896.218387:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98918400 chunk: [98918400, 98922496) 98922496, 4096 00020000:00200000:0.0:1775764896.218390:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.218401:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24150, 24150] 00000008:00010000:0.0:1775764896.218407:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.218409:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.218418:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218427:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98918400 98922495) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218436:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218526:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218533:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.218542:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98918400, 98922496) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.218548:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24150 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.218566:0:18302:0:(rw26.c:858:ll_write_end()) pos 98918400, len 4096, copied 4096 00000080:00200000:0.0:1775764896.218571:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b15aa490[3 00000000b99c1f5d 2 1 000000003a48b37d] 00000080:00200000:0.0:1775764896.218574:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000896f90ac 17ffffc0001001 3:0 ffff90c212812000 24150 lru 00000080:00200000:0.0:1775764896.218579:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d450489b 24150: 1< 0 - - > 2< 98918400 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1695744 - | + - + - > 5< + - + - | 0 - | 3670 - -> 00000080:00200000:0.0:1775764896.218581:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b15aa490 00000080:00200000:0.0:1775764896.218582:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.218584:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.218591:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.218598:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.218604:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.218611:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.218622:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.218624:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.218626:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.218636:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.218638:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.218640:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.218644:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218651:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.218659:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.218661:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.218666:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.218669:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24150, 24150] 00000080:00200000:0.0:1775764896.218679:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.218682:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.218709:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98922496 i_blocks 90112 00000080:00200000:0.0:1775764896.218712:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.218727:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98922496, count: 4096 00000080:00200000:0.0:1775764896.218737:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98922496, bytes: 4096 00000080:00200000:0.0:1775764896.218739:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.218743:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24152, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.218744:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98922496, 98926592) 0 0 00000080:00200000:0.0:1775764896.218747:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.218751:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24151, 24151] 00020000:00200000:0.0:1775764896.218755:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98922496 chunk: [98922496, 98926592) 98926592, 4096 00020000:00200000:0.0:1775764896.218757:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.218767:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24151, 24151] 00000008:00010000:0.0:1775764896.218772:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.218774:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.218782:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218790:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98922496 98926591) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218798:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218805:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218811:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.218818:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98922496, 98926592) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.218823:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24151 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.218840:0:18302:0:(rw26.c:858:ll_write_end()) pos 98922496, len 4096, copied 4096 00000080:00200000:0.0:1775764896.218844:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008711f217[3 00000000b99c1f5d 2 1 000000003a48b37d] 00000080:00200000:0.0:1775764896.218846:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006f16556f 17ffffc0001001 3:0 ffff90c212812ee0 24151 lru 00000080:00200000:0.0:1775764896.218852:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000139563b4 24151: 1< 0 - - > 2< 98922496 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1691648 - | + - + - > 5< + - + - | 0 - | 3671 - -> 00000080:00200000:0.0:1775764896.218853:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008711f217 00000080:00200000:0.0:1775764896.218854:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.218857:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.218864:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.218895:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.218902:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.218909:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.218919:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.218922:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.218924:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.218933:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.218935:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.218938:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.218941:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.218949:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.218958:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.218960:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.218965:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.218968:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24151, 24151] 00000080:00200000:0.0:1775764896.218978:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.218980:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.219010:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98926592 i_blocks 90112 00000080:00200000:0.0:1775764896.219014:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.219029:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98926592, count: 4096 00000080:00200000:0.0:1775764896.219040:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98926592, bytes: 4096 00000080:00200000:0.0:1775764896.219043:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.219046:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24153, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.219048:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98926592, 98930688) 0 0 00000080:00200000:0.0:1775764896.219050:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.219055:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24152, 24152] 00020000:00200000:0.0:1775764896.219058:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98926592 chunk: [98926592, 98930688) 98930688, 4096 00020000:00200000:0.0:1775764896.219060:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.219072:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24152, 24152] 00000008:00010000:0.0:1775764896.219077:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.219080:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.219087:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.219097:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98926592 98930687) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.219105:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.219129:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.219136:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.219143:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98926592, 98930688) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.219149:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24152 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.219165:0:18302:0:(rw26.c:858:ll_write_end()) pos 98926592, len 4096, copied 4096 00000080:00200000:0.0:1775764896.219169:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005f89165e[3 00000000b99c1f5d 2 1 000000003a48b37d] 00000080:00200000:0.0:1775764896.219172:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000064847efa 17ffffc0001001 3:0 ffff90c2128120e0 24152 lru 00000080:00200000:0.0:1775764896.219178:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000af084fd2 24152: 1< 0 - - > 2< 98926592 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1687552 - | + - + - > 5< + - + - | 0 - | 3672 - -> 00000080:00200000:0.0:1775764896.219180:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005f89165e 00000080:00200000:0.0:1775764896.219181:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.219183:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.219191:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.219198:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.219204:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.219211:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.219226:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.219229:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.219231:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.219241:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.219243:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.219246:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.219250:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.219618:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.219627:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.219630:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.219635:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.219639:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24152, 24152] 00000080:00200000:0.0:1775764896.219651:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.219654:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.219687:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98930688 i_blocks 90112 00000080:00200000:0.0:1775764896.219691:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.219708:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98930688, count: 4096 00000080:00200000:0.0:1775764896.219719:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98930688, bytes: 4096 00000080:00200000:0.0:1775764896.219722:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.219725:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24154, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.219796:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98930688, 98934784) 0 0 00000080:00200000:0.0:1775764896.219800:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.219807:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24153, 24153] 00020000:00200000:0.0:1775764896.219811:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98930688 chunk: [98930688, 98934784) 98934784, 4096 00020000:00200000:0.0:1775764896.219814:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.219829:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24153, 24153] 00000008:00010000:0.0:1775764896.219835:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.219838:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.219847:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.219857:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98930688 98934783) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.219865:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.219896:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.219903:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.219912:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98930688, 98934784) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.219918:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24153 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.219940:0:18302:0:(rw26.c:858:ll_write_end()) pos 98930688, len 4096, copied 4096 00000080:00200000:0.0:1775764896.219944:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004a308845[3 00000000b99c1f5d 2 1 000000003a48b37d] 00000080:00200000:0.0:1775764896.219947:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003844ae42 17ffffc0001001 3:0 ffff90c2128121c0 24153 lru 00000080:00200000:0.0:1775764896.219953:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e90a8e47 24153: 1< 0 - - > 2< 98930688 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1683456 - | + - + - > 5< + - + - | 0 - | 3673 - -> 00000080:00200000:0.0:1775764896.219955:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004a308845 00000080:00200000:0.0:1775764896.219956:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.219959:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.219969:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.219977:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.219984:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.219992:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.220004:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.220008:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.220010:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.220021:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.220024:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.220026:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.220030:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220037:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.220045:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.220047:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.220052:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.220054:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24153, 24153] 00000080:00200000:0.0:1775764896.220066:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.220069:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.220096:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98934784 i_blocks 90112 00000080:00200000:0.0:1775764896.220100:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.220127:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98934784, count: 4096 00000080:00200000:0.0:1775764896.220138:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98934784, bytes: 4096 00000080:00200000:0.0:1775764896.220141:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.220145:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24155, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.220146:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98934784, 98938880) 0 0 00000080:00200000:0.0:1775764896.220148:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.220154:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24154, 24154] 00020000:00200000:0.0:1775764896.220156:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98934784 chunk: [98934784, 98938880) 98938880, 4096 00020000:00200000:0.0:1775764896.220159:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.220170:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24154, 24154] 00000008:00010000:0.0:1775764896.220175:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.220178:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.220185:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220194:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98934784 98938879) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220202:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220210:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220217:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.220226:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98934784, 98938880) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.220231:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24154 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.220249:0:18302:0:(rw26.c:858:ll_write_end()) pos 98934784, len 4096, copied 4096 00000080:00200000:0.0:1775764896.220253:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e285a7c7[3 00000000b99c1f5d 2 1 000000003a48b37d] 00000080:00200000:0.0:1775764896.220256:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005c336ca3 17ffffc0001001 3:0 ffff90c2128127e0 24154 lru 00000080:00200000:0.0:1775764896.220260:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b2664544 24154: 1< 0 - - > 2< 98934784 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1679360 - | + - + - > 5< + - + - | 0 - | 3674 - -> 00000080:00200000:0.0:1775764896.220262:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e285a7c7 00000080:00200000:0.0:1775764896.220264:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.220265:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.220273:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.220281:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.220288:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.220295:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.220306:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.220308:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.220310:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.220320:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.220322:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.220325:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.220328:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220336:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.220344:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.220346:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.220351:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.220353:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24154, 24154] 00000080:00200000:0.0:1775764896.220364:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.220367:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.220397:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98938880 i_blocks 90112 00000080:00200000:0.0:1775764896.220401:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.220417:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98938880, count: 4096 00000080:00200000:0.0:1775764896.220428:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98938880, bytes: 4096 00000080:00200000:0.0:1775764896.220431:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.220434:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24156, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.220518:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98938880, 98942976) 0 0 00000080:00200000:0.0:1775764896.220523:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.220529:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24155, 24155] 00020000:00200000:0.0:1775764896.220532:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98938880 chunk: [98938880, 98942976) 98942976, 4096 00020000:00200000:0.0:1775764896.220535:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.220549:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24155, 24155] 00000008:00010000:0.0:1775764896.220555:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.220558:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.220565:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220575:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98938880 98942975) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220584:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220592:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220599:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.220606:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98938880, 98942976) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.220612:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24155 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.220628:0:18302:0:(rw26.c:858:ll_write_end()) pos 98938880, len 4096, copied 4096 00000080:00200000:0.0:1775764896.220633:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000007462d26[3 00000000b99c1f5d 2 1 000000003a48b37d] 00000080:00200000:0.0:1775764896.220636:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e69586fb 17ffffc0001001 3:0 ffff90c2128129a0 24155 lru 00000080:00200000:0.0:1775764896.220642:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000116a4969 24155: 1< 0 - - > 2< 98938880 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1675264 - | + - + - > 5< + - + - | 0 - | 3675 - -> 00000080:00200000:0.0:1775764896.220644:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000007462d26 00000080:00200000:0.0:1775764896.220645:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.220647:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.220655:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.220663:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ed85ac00 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.220670:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.220682:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.220694:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.220696:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.220698:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.220709:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.220711:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.220714:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.220718:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.220725:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.220822:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1038 (ffff90c210c81f38/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.220825:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81f38/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.220830:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.220833:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24155, 24155] 00000080:00200000:0.0:1775764896.220844:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.220847:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.220972:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98942976 i_blocks 90112 00000080:00200000:1.0:1775764896.220977:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.220998:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98942976, count: 4096 00000080:00200000:1.0:1775764896.221015:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98942976, bytes: 4096 00000080:00200000:1.0:1775764896.221019:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.221023:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24157, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.221024:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98942976, 98947072) 0 0 00000080:00200000:1.0:1775764896.221026:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.221032:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24156, 24156] 00020000:00200000:1.0:1775764896.221035:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98942976 chunk: [98942976, 98947072) 98947072, 4096 00020000:00200000:1.0:1775764896.221037:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.221052:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24156, 24156] 00000008:00010000:1.0:1775764896.221059:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.221061:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.221069:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221077:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98942976 98947071) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221085:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221092:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221105:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.221125:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98942976, 98947072) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.221132:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24156 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.221150:0:18302:0:(rw26.c:858:ll_write_end()) pos 98942976, len 4096, copied 4096 00000080:00200000:1.0:1775764896.221154:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fde7869a[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.221157:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000591cb50c 17ffffc0001001 3:0 ffff90c212813380 24156 lru 00000080:00200000:1.0:1775764896.221163:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007e1c4f48 24156: 1< 0 - - > 2< 98942976 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1671168 - | + - + - > 5< + - + - | 0 - | 3676 - -> 00000080:00200000:1.0:1775764896.221165:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fde7869a 00000080:00200000:1.0:1775764896.221166:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.221168:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.221176:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.221184:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.221191:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.221199:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.221211:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.221214:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.221217:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.221225:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.221227:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.221230:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.221234:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221241:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.221249:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.221251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.221256:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.221259:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24156, 24156] 00000080:00200000:1.0:1775764896.221275:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.221278:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.221282:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98947072 i_blocks 90112 00000080:00200000:1.0:1775764896.221285:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.221300:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98947072, count: 4096 00000080:00200000:1.0:1775764896.221311:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98947072, bytes: 4096 00000080:00200000:1.0:1775764896.221314:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.221317:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24158, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.221319:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98947072, 98951168) 0 0 00000080:00200000:1.0:1775764896.221321:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.221326:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24157, 24157] 00020000:00200000:1.0:1775764896.221329:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98947072 chunk: [98947072, 98951168) 98951168, 4096 00020000:00200000:1.0:1775764896.221331:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.221343:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24157, 24157] 00000008:00010000:1.0:1775764896.221348:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.221350:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.221358:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221366:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98947072 98951167) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221375:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221382:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221388:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.221396:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98947072, 98951168) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.221401:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24157 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.221417:0:18302:0:(rw26.c:858:ll_write_end()) pos 98947072, len 4096, copied 4096 00000080:00200000:1.0:1775764896.221421:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003b795817[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.221423:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a1f8fcf5 17ffffc0001001 3:0 ffff90c212813620 24157 lru 00000080:00200000:1.0:1775764896.221429:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006e73fe08 24157: 1< 0 - - > 2< 98947072 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1667072 - | + - + - > 5< + - + - | 0 - | 3677 - -> 00000080:00200000:1.0:1775764896.221431:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003b795817 00000080:00200000:1.0:1775764896.221431:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.221433:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.221524:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.221533:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.221540:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.221547:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.221559:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.221563:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.221565:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.221571:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.221573:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.221576:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.221580:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221588:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.221597:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.221599:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.221603:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.221606:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24157, 24157] 00000080:00200000:1.0:1775764896.221617:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.221620:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.221625:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98951168 i_blocks 90112 00000080:00200000:1.0:1775764896.221628:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.221643:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98951168, count: 4096 00000080:00200000:1.0:1775764896.221655:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98951168, bytes: 4096 00000080:00200000:1.0:1775764896.221658:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.221661:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24159, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.221663:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98951168, 98955264) 0 0 00000080:00200000:1.0:1775764896.221665:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.221670:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24158, 24158] 00020000:00200000:1.0:1775764896.221674:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98951168 chunk: [98951168, 98955264) 98955264, 4096 00020000:00200000:1.0:1775764896.221676:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.221688:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24158, 24158] 00000008:00010000:1.0:1775764896.221693:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.221695:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.221703:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221712:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98951168 98955263) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221719:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221726:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221741:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.221749:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98951168, 98955264) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.221755:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24158 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.221773:0:18302:0:(rw26.c:858:ll_write_end()) pos 98951168, len 4096, copied 4096 00000080:00200000:1.0:1775764896.221777:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000298e5e2a[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.221781:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000e491578 17ffffc0001001 3:0 ffff90c212813e00 24158 lru 00000080:00200000:1.0:1775764896.221786:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004fe5a9c5 24158: 1< 0 - - > 2< 98951168 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1662976 - | + - + - > 5< + - + - | 0 - | 3678 - -> 00000080:00200000:1.0:1775764896.221788:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000298e5e2a 00000080:00200000:1.0:1775764896.221789:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.221791:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.221799:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.221807:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.221813:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.221821:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.221833:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.221835:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.221838:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.221843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.221846:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.221848:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.221852:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.221859:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.221868:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.221895:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.221899:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.221902:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24158, 24158] 00000080:00200000:1.0:1775764896.221913:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.221916:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.221921:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98955264 i_blocks 90112 00000080:00200000:1.0:1775764896.221924:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.221939:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98955264, count: 4096 00000080:00200000:1.0:1775764896.221951:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98955264, bytes: 4096 00000080:00200000:1.0:1775764896.221953:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.221957:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24160, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.221959:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98955264, 98959360) 0 0 00000080:00200000:1.0:1775764896.221962:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.221967:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24159, 24159] 00020000:00200000:1.0:1775764896.221970:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98955264 chunk: [98955264, 98959360) 98959360, 4096 00020000:00200000:1.0:1775764896.221972:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.221984:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24159, 24159] 00000008:00010000:1.0:1775764896.221989:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.221992:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.222000:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222009:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98955264 98959359) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222018:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222032:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222039:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.222047:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98955264, 98959360) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.222053:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24159 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.222069:0:18302:0:(rw26.c:858:ll_write_end()) pos 98955264, len 4096, copied 4096 00000080:00200000:1.0:1775764896.222073:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004df56c3f[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.222076:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000040bbced5 17ffffc0001001 3:0 ffff90c212813540 24159 lru 00000080:00200000:1.0:1775764896.222082:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f26be64e 24159: 1< 0 - - > 2< 98955264 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1658880 - | + - + - > 5< + - + - | 0 - | 3679 - -> 00000080:00200000:1.0:1775764896.222084:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004df56c3f 00000080:00200000:1.0:1775764896.222085:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.222087:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.222095:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.222102:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.222107:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.222132:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.222150:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.222153:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.222155:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.222161:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.222163:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.222165:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.222169:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222176:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.222184:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.222187:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.222191:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.222194:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24159, 24159] 00000080:00200000:1.0:1775764896.222204:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.222208:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.222212:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98959360 i_blocks 90112 00000080:00200000:1.0:1775764896.222215:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.222231:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98959360, count: 4096 00000080:00200000:1.0:1775764896.222242:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98959360, bytes: 4096 00000080:00200000:1.0:1775764896.222245:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.222248:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24161, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.222249:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98959360, 98963456) 0 0 00000080:00200000:1.0:1775764896.222251:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.222256:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24160, 24160] 00020000:00200000:1.0:1775764896.222259:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98959360 chunk: [98959360, 98963456) 98963456, 4096 00020000:00200000:1.0:1775764896.222261:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.222271:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24160, 24160] 00000008:00010000:1.0:1775764896.222276:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.222279:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.222286:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222294:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98959360 98963455) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222302:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222309:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222316:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.222324:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98959360, 98963456) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.222329:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24160 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.222345:0:18302:0:(rw26.c:858:ll_write_end()) pos 98959360, len 4096, copied 4096 00000080:00200000:1.0:1775764896.222350:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ca373892[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.222353:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e1339576 17ffffc0001001 3:0 ffff90c212813460 24160 lru 00000080:00200000:1.0:1775764896.222358:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000caf5b855 24160: 1< 0 - - > 2< 98959360 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1654784 - | + - + - > 5< + - + - | 0 - | 3680 - -> 00000080:00200000:1.0:1775764896.222360:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ca373892 00000080:00200000:1.0:1775764896.222361:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.222364:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.222372:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.222380:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.222387:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.222394:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.222406:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.222409:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.222411:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.222417:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.222419:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.222421:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.222425:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.222432:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.223180:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.223183:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.223189:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.223193:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24160, 24160] 00000080:00200000:1.0:1775764896.223207:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.223210:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.223217:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98963456 i_blocks 90112 00000080:00200000:1.0:1775764896.223221:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.223239:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98963456, count: 4096 00000080:00200000:1.0:1775764896.223252:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98963456, bytes: 4096 00000080:00200000:1.0:1775764896.223255:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.223260:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24162, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.223262:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98963456, 98967552) 0 0 00000080:00200000:1.0:1775764896.223264:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.223271:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24161, 24161] 00020000:00200000:1.0:1775764896.223274:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98963456 chunk: [98963456, 98967552) 98967552, 4096 00020000:00200000:1.0:1775764896.223276:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.223290:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24161, 24161] 00000008:00010000:1.0:1775764896.223297:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.223300:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.223310:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223319:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98963456 98967551) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223328:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223336:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223343:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.223352:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98963456, 98967552) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.223358:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24161 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.223382:0:18302:0:(rw26.c:858:ll_write_end()) pos 98963456, len 4096, copied 4096 00000080:00200000:1.0:1775764896.223387:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b0881836[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.223390:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004a9bc662 17ffffc0001001 3:0 ffff90c212813a80 24161 lru 00000080:00200000:1.0:1775764896.223396:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000117dd43c 24161: 1< 0 - - > 2< 98963456 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1650688 - | + - + - > 5< + - + - | 0 - | 3681 - -> 00000080:00200000:1.0:1775764896.223398:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b0881836 00000080:00200000:1.0:1775764896.223400:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.223402:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.223412:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.223420:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.223428:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.223436:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.223530:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.223534:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.223536:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.223554:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.223557:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.223559:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.223563:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223571:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.223579:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.223582:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.223586:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.223589:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24161, 24161] 00000080:00200000:1.0:1775764896.223600:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.223603:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.223640:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98967552 i_blocks 90112 00000080:00200000:1.0:1775764896.223643:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.223655:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98967552, count: 4096 00000080:00200000:1.0:1775764896.223661:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98967552, bytes: 4096 00000080:00200000:1.0:1775764896.223663:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.223666:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24163, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.223667:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98967552, 98971648) 0 0 00000080:00200000:1.0:1775764896.223669:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.223673:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24162, 24162] 00020000:00200000:1.0:1775764896.223675:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98967552 chunk: [98967552, 98971648) 98971648, 4096 00020000:00200000:1.0:1775764896.223677:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.223684:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24162, 24162] 00000008:00010000:1.0:1775764896.223688:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.223690:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.223695:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223701:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98967552 98971647) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223707:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223712:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223716:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.223721:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98967552, 98971648) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.223724:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24162 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.223803:0:18302:0:(rw26.c:858:ll_write_end()) pos 98967552, len 4096, copied 4096 00000080:00200000:1.0:1775764896.223808:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000eb9c90cb[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.223812:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d7032eb0 17ffffc0001001 3:0 ffff90c212813700 24162 lru 00000080:00200000:1.0:1775764896.223817:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000fcad63fa 24162: 1< 0 - - > 2< 98967552 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1646592 - | + - + - > 5< + - + - | 0 - | 3682 - -> 00000080:00200000:1.0:1775764896.223819:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000eb9c90cb 00000080:00200000:1.0:1775764896.223821:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.223823:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.223831:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.223839:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.223846:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.223855:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.223864:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.223866:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.223867:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.223889:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.223891:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.223893:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.223896:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.223901:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.223906:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.223907:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.223910:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.223912:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24162, 24162] 00000080:00200000:1.0:1775764896.223919:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.223921:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.223941:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98971648 i_blocks 90112 00000080:00200000:1.0:1775764896.223944:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.223960:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98971648, count: 4096 00000080:00200000:1.0:1775764896.223971:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98971648, bytes: 4096 00000080:00200000:1.0:1775764896.223973:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.223978:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24164, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.223979:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98971648, 98975744) 0 0 00000080:00200000:1.0:1775764896.223981:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.223987:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24163, 24163] 00020000:00200000:1.0:1775764896.223990:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98971648 chunk: [98971648, 98975744) 98975744, 4096 00020000:00200000:1.0:1775764896.223992:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.224004:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24163, 24163] 00000008:00010000:1.0:1775764896.224009:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.224011:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.224019:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224029:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98971648 98975743) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224037:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224044:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224051:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.224059:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98971648, 98975744) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.224064:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24163 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.224081:0:18302:0:(rw26.c:858:ll_write_end()) pos 98971648, len 4096, copied 4096 00000080:00200000:1.0:1775764896.224085:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003403ed07[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.224088:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000671a78b0 17ffffc0001001 3:0 ffff90c212813c40 24163 lru 00000080:00200000:1.0:1775764896.224094:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000032bc7e97 24163: 1< 0 - - > 2< 98971648 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1642496 - | + - + - > 5< + - + - | 0 - | 3683 - -> 00000080:00200000:1.0:1775764896.224096:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003403ed07 00000080:00200000:1.0:1775764896.224098:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.224100:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.224108:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.224127:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.224134:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.224140:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.224148:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.224150:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.224152:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.224159:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.224160:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.224162:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.224165:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224170:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.224175:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.224177:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.224179:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.224182:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24163, 24163] 00000080:00200000:1.0:1775764896.224189:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.224191:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.224211:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98975744 i_blocks 90112 00000080:00200000:1.0:1775764896.224213:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.224224:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98975744, count: 4096 00000080:00200000:1.0:1775764896.224230:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98975744, bytes: 4096 00000080:00200000:1.0:1775764896.224232:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.224235:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24165, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.224236:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98975744, 98979840) 0 0 00000080:00200000:1.0:1775764896.224237:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.224241:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24164, 24164] 00020000:00200000:1.0:1775764896.224243:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98975744 chunk: [98975744, 98979840) 98979840, 4096 00020000:00200000:1.0:1775764896.224245:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.224251:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24164, 24164] 00000008:00010000:1.0:1775764896.224254:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.224256:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.224261:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224267:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98975744 98979839) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224272:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224277:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224285:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.224290:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98975744, 98979840) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.224293:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24164 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.224306:0:18302:0:(rw26.c:858:ll_write_end()) pos 98975744, len 4096, copied 4096 00000080:00200000:1.0:1775764896.224309:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d947b1df[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.224311:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d069cc15 17ffffc0001001 3:0 ffff90c2128138c0 24164 lru 00000080:00200000:1.0:1775764896.224315:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000151188b6 24164: 1< 0 - - > 2< 98975744 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1638400 - | + - + - > 5< + - + - | 0 - | 3684 - -> 00000080:00200000:1.0:1775764896.224316:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d947b1df 00000080:00200000:1.0:1775764896.224317:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.224318:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.224323:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.224329:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.224337:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.224344:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.224355:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.224358:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.224360:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.224369:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.224372:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.224374:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.224378:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224386:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.224395:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.224397:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.224401:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.224404:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24164, 24164] 00000080:00200000:1.0:1775764896.224412:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.224801:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.224836:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98979840 i_blocks 90112 00000080:00200000:1.0:1775764896.224841:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.224857:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98979840, count: 4096 00000080:00200000:1.0:1775764896.224866:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98979840, bytes: 4096 00000080:00200000:1.0:1775764896.224868:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.224889:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24166, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.224891:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98979840, 98983936) 0 0 00000080:00200000:1.0:1775764896.224892:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.224897:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24165, 24165] 00020000:00200000:1.0:1775764896.224899:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98979840 chunk: [98979840, 98983936) 98983936, 4096 00020000:00200000:1.0:1775764896.224901:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.224909:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24165, 24165] 00000008:00010000:1.0:1775764896.224913:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.224915:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.224922:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224928:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98979840 98983935) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224934:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224938:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.224944:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.224952:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98979840, 98983936) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.224957:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24165 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.224976:0:18302:0:(rw26.c:858:ll_write_end()) pos 98979840, len 4096, copied 4096 00000080:00200000:1.0:1775764896.224980:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000076b50b85[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.224983:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008d66d164 17ffffc0001001 3:0 ffff90c212813b60 24165 lru 00000080:00200000:1.0:1775764896.224988:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f3af32ba 24165: 1< 0 - - > 2< 98979840 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1634304 - | + - + - > 5< + - + - | 0 - | 3685 - -> 00000080:00200000:1.0:1775764896.224990:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000076b50b85 00000080:00200000:1.0:1775764896.224991:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.224993:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.225002:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.225009:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.225016:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.225024:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.225036:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.225039:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.225041:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.225052:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.225055:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.225057:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.225061:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225070:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.225078:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.225081:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.225085:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.225088:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24165, 24165] 00000080:00200000:1.0:1775764896.225098:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.225107:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.225292:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98983936 i_blocks 90112 00000080:00200000:1.0:1775764896.225296:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.225313:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98983936, count: 4096 00000080:00200000:1.0:1775764896.225323:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98983936, bytes: 4096 00000080:00200000:1.0:1775764896.225326:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.225330:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24167, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.225331:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98983936, 98988032) 0 0 00000080:00200000:1.0:1775764896.225334:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.225339:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24166, 24166] 00020000:00200000:1.0:1775764896.225342:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98983936 chunk: [98983936, 98988032) 98988032, 4096 00020000:00200000:1.0:1775764896.225345:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.225357:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24166, 24166] 00000008:00010000:1.0:1775764896.225363:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.225366:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.225373:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225381:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98983936 98988031) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225390:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225396:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225403:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.225411:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98983936, 98988032) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.225416:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24166 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.225434:0:18302:0:(rw26.c:858:ll_write_end()) pos 98983936, len 4096, copied 4096 00000080:00200000:1.0:1775764896.225450:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fd7c58bb[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.225452:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000010670c0d 17ffffc0001001 3:0 ffff90c212813000 24166 lru 00000080:00200000:1.0:1775764896.225458:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000461d82de 24166: 1< 0 - - > 2< 98983936 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1630208 - | + - + - > 5< + - + - | 0 - | 3686 - -> 00000080:00200000:1.0:1775764896.225460:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fd7c58bb 00000080:00200000:1.0:1775764896.225461:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.225463:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.225471:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.225478:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.225484:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.225491:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.225502:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.225504:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.225506:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.225516:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.225519:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.225521:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.225525:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225532:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.225541:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.225544:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.225548:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.225551:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24166, 24166] 00000080:00200000:1.0:1775764896.225561:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.225565:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.225595:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98988032 i_blocks 90112 00000080:00200000:1.0:1775764896.225599:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.225615:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98988032, count: 4096 00000080:00200000:1.0:1775764896.225625:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98988032, bytes: 4096 00000080:00200000:1.0:1775764896.225628:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.225632:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24168, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.225633:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98988032, 98992128) 0 0 00000080:00200000:1.0:1775764896.225636:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.225641:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24167, 24167] 00020000:00200000:1.0:1775764896.225644:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98988032 chunk: [98988032, 98992128) 98992128, 4096 00020000:00200000:1.0:1775764896.225646:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.225657:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24167, 24167] 00000008:00010000:1.0:1775764896.225663:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.225665:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.225673:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225682:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98988032 98992127) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225690:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225704:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225711:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.225719:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98988032, 98992128) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.225724:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24167 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.225755:0:18302:0:(rw26.c:858:ll_write_end()) pos 98988032, len 4096, copied 4096 00000080:00200000:1.0:1775764896.225759:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c45005f0[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.225766:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ee5af10c 17ffffc0001001 3:0 ffff90c212813ee0 24167 lru 00000080:00200000:1.0:1775764896.225772:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a60f3a01 24167: 1< 0 - - > 2< 98988032 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1626112 - | + - + - > 5< + - + - | 0 - | 3687 - -> 00000080:00200000:1.0:1775764896.225774:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c45005f0 00000080:00200000:1.0:1775764896.225776:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.225778:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.225786:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.225794:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.225800:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.225807:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.225823:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.225826:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.225828:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.225838:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.225840:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.225843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.225847:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.225855:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.225863:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.225865:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.225901:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.225905:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24167, 24167] 00000080:00200000:1.0:1775764896.225916:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.225919:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.225947:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98992128 i_blocks 90112 00000080:00200000:1.0:1775764896.225951:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.225968:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98992128, count: 4096 00000080:00200000:1.0:1775764896.225978:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98992128, bytes: 4096 00000080:00200000:1.0:1775764896.225981:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.225984:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24169, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.225985:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98992128, 98996224) 0 0 00000080:00200000:1.0:1775764896.225987:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.225992:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24168, 24168] 00020000:00200000:1.0:1775764896.225995:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98992128 chunk: [98992128, 98996224) 98996224, 4096 00020000:00200000:1.0:1775764896.225997:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.226006:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24168, 24168] 00000008:00010000:1.0:1775764896.226011:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.226013:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.226020:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226027:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98992128 98996223) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226034:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226041:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226048:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.226055:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98992128, 98996224) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.226060:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24168 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.226083:0:18302:0:(rw26.c:858:ll_write_end()) pos 98992128, len 4096, copied 4096 00000080:00200000:1.0:1775764896.226088:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007a513349[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.226091:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003c0ef9b9 17ffffc0001021 2:0 ffff90c2128130e0 24168 lru 00000080:00200000:1.0:1775764896.226096:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008dd8b0eb 24168: 1< 0 - - > 2< 98992128 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1622016 - | + - + - > 5< + - + - | 0 - | 3688 - -> 00000080:00200000:1.0:1775764896.226098:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007a513349 00000080:00200000:1.0:1775764896.226100:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.226102:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.226110:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226128:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226135:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226143:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.226157:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.226160:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.226162:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.226172:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.226174:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.226177:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.226181:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226188:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.226196:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.226198:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.226202:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.226204:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24168, 24168] 00000080:00200000:1.0:1775764896.226214:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.226217:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.226244:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 98996224 i_blocks 90112 00000080:00200000:1.0:1775764896.226248:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.226264:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 98996224, count: 4096 00000080:00200000:1.0:1775764896.226275:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 98996224, bytes: 4096 00000080:00200000:1.0:1775764896.226278:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.226282:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24170, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.226283:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [98996224, 99000320) 0 0 00000080:00200000:1.0:1775764896.226286:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.226291:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24169, 24169] 00020000:00200000:1.0:1775764896.226294:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 98996224 chunk: [98996224, 99000320) 99000320, 4096 00020000:00200000:1.0:1775764896.226296:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.226307:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24169, 24169] 00000008:00010000:1.0:1775764896.226313:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.226315:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.226323:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226331:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (98996224 99000319) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226340:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226347:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226355:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.226363:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [98996224, 99000320) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.226368:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24169 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.226383:0:18302:0:(rw26.c:858:ll_write_end()) pos 98996224, len 4096, copied 4096 00000080:00200000:1.0:1775764896.226387:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000bbd2b925[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.226390:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000066cbc3bc 17ffffc0001001 3:0 ffff90c2128131c0 24169 lru 00000080:00200000:1.0:1775764896.226396:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000084c65b17 24169: 1< 0 - - > 2< 98996224 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1617920 - | + - + - > 5< + - + - | 0 - | 3689 - -> 00000080:00200000:1.0:1775764896.226398:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000bbd2b925 00000080:00200000:1.0:1775764896.226399:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.226401:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.226409:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226417:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226424:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226431:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.226443:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.226446:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.226449:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.226459:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.226462:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.226465:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.226469:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226476:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.226483:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.226485:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.226490:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.226492:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24169, 24169] 00000080:00200000:1.0:1775764896.226503:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.226506:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.226534:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99000320 i_blocks 90112 00000080:00200000:1.0:1775764896.226538:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.226554:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99000320, count: 4096 00000080:00200000:1.0:1775764896.226565:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99000320, bytes: 4096 00000080:00200000:1.0:1775764896.226567:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.226571:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24171, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.226573:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99000320, 99004416) 0 0 00000080:00200000:1.0:1775764896.226575:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.226580:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24170, 24170] 00020000:00200000:1.0:1775764896.226584:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99000320 chunk: [99000320, 99004416) 99004416, 4096 00020000:00200000:1.0:1775764896.226586:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.226598:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24170, 24170] 00000008:00010000:1.0:1775764896.226603:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.226605:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.226613:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226622:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99000320 99004415) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226631:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226639:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226646:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.226653:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99000320, 99004416) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.226658:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24170 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.226675:0:18302:0:(rw26.c:858:ll_write_end()) pos 99000320, len 4096, copied 4096 00000080:00200000:1.0:1775764896.226680:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000066b6ddf1[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.226683:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fa214be6 17ffffc0001001 3:0 ffff90c2128137e0 24170 lru 00000080:00200000:1.0:1775764896.226688:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000064717873 24170: 1< 0 - - > 2< 99000320 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1613824 - | + - + - > 5< + - + - | 0 - | 3690 - -> 00000080:00200000:1.0:1775764896.226691:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000066b6ddf1 00000080:00200000:1.0:1775764896.226692:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.226694:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.226702:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226708:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226712:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226721:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.226729:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.226731:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.226732:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.226739:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.226740:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.226742:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.226745:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226750:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.226755:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.226756:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.226758:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.226760:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24170, 24170] 00000080:00200000:1.0:1775764896.226767:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.226770:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.226797:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99004416 i_blocks 90112 00000080:00200000:1.0:1775764896.226801:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.226816:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99004416, count: 4096 00000080:00200000:1.0:1775764896.226827:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99004416, bytes: 4096 00000080:00200000:1.0:1775764896.226830:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.226834:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24172, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.226835:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99004416, 99008512) 0 0 00000080:00200000:1.0:1775764896.226838:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.226843:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24171, 24171] 00020000:00200000:1.0:1775764896.226845:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99004416 chunk: [99004416, 99008512) 99008512, 4096 00020000:00200000:1.0:1775764896.226847:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.226855:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24171, 24171] 00000008:00010000:1.0:1775764896.226858:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.226860:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.226865:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226890:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99004416 99008511) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226896:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226901:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.226905:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.226909:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99004416, 99008512) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.226913:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24171 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.226925:0:18302:0:(rw26.c:858:ll_write_end()) pos 99004416, len 4096, copied 4096 00000080:00200000:1.0:1775764896.226928:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f0eedaf7[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.226930:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a509e62a 17ffffc0001001 3:0 ffff90c2128139a0 24171 lru 00000080:00200000:1.0:1775764896.226933:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000054cc6da7 24171: 1< 0 - - > 2< 99004416 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1609728 - | + - + - > 5< + - + - | 0 - | 3691 - -> 00000080:00200000:1.0:1775764896.226935:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f0eedaf7 00000080:00200000:1.0:1775764896.226936:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.226937:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.226944:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226952:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226959:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.226966:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.226977:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.226980:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.226982:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.226992:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.226995:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.226997:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.227001:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227010:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.227018:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.227021:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.227026:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.227028:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24171, 24171] 00000080:00200000:1.0:1775764896.227035:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.227038:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.227057:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99008512 i_blocks 90112 00000080:00200000:1.0:1775764896.227060:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.227071:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99008512, count: 4096 00000080:00200000:1.0:1775764896.227077:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99008512, bytes: 4096 00000080:00200000:1.0:1775764896.227079:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.227081:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24173, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.227082:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99008512, 99012608) 0 0 00000080:00200000:1.0:1775764896.227084:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.227087:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24172, 24172] 00020000:00200000:1.0:1775764896.227089:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99008512 chunk: [99008512, 99012608) 99012608, 4096 00020000:00200000:1.0:1775764896.227091:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.227097:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24172, 24172] 00000008:00010000:1.0:1775764896.227100:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.227102:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.227108:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227127:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99008512 99012607) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227136:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227144:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227154:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.227162:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99008512, 99012608) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.227168:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24172 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.227195:0:18302:0:(rw26.c:858:ll_write_end()) pos 99008512, len 4096, copied 4096 00000080:00200000:1.0:1775764896.227199:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000073ee5155[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.227202:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000f612171 17ffffc0001001 3:0 ffff90c212814540 24172 lru 00000080:00200000:1.0:1775764896.227208:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a939562e 24172: 1< 0 - - > 2< 99008512 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1605632 - | + - + - > 5< + - + - | 0 - | 3692 - -> 00000080:00200000:1.0:1775764896.227210:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000073ee5155 00000080:00200000:1.0:1775764896.227211:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.227214:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.227222:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227234:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227241:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227248:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.227261:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.227263:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.227266:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.227276:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.227278:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.227280:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.227284:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227290:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.227298:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.227300:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.227304:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.227306:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24172, 24172] 00000080:00200000:1.0:1775764896.227316:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.227319:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.227348:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99012608 i_blocks 90112 00000080:00200000:1.0:1775764896.227351:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.227367:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99012608, count: 4096 00000080:00200000:1.0:1775764896.227377:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99012608, bytes: 4096 00000080:00200000:1.0:1775764896.227380:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.227384:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24174, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.227385:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99012608, 99016704) 0 0 00000080:00200000:1.0:1775764896.227388:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.227393:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24173, 24173] 00020000:00200000:1.0:1775764896.227396:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99012608 chunk: [99012608, 99016704) 99016704, 4096 00020000:00200000:1.0:1775764896.227398:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.227409:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24173, 24173] 00000008:00010000:1.0:1775764896.227414:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.227417:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.227426:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227435:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99012608 99016703) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227443:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227450:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227456:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.227464:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99012608, 99016704) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.227470:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24173 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.227487:0:18302:0:(rw26.c:858:ll_write_end()) pos 99012608, len 4096, copied 4096 00000080:00200000:1.0:1775764896.227491:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000068ac41be[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.227494:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000073497aed 17ffffc0001001 3:0 ffff90c212814460 24173 lru 00000080:00200000:1.0:1775764896.227499:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000fb28729e 24173: 1< 0 - - > 2< 99012608 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1601536 - | + - + - > 5< + - + - | 0 - | 3693 - -> 00000080:00200000:1.0:1775764896.227502:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000068ac41be 00000080:00200000:1.0:1775764896.227503:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.227505:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.227512:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227518:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227523:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227527:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.227534:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.227536:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.227538:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.227547:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.227549:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.227551:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.227555:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227562:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.227569:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.227571:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.227576:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.227578:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24173, 24173] 00000080:00200000:1.0:1775764896.227589:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.227592:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.227617:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99016704 i_blocks 90112 00000080:00200000:1.0:1775764896.227620:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.227631:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99016704, count: 4096 00000080:00200000:1.0:1775764896.227637:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99016704, bytes: 4096 00000080:00200000:1.0:1775764896.227639:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.227641:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24175, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.227642:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99016704, 99020800) 0 0 00000080:00200000:1.0:1775764896.227644:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.227648:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24174, 24174] 00020000:00200000:1.0:1775764896.227650:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99016704 chunk: [99016704, 99020800) 99020800, 4096 00020000:00200000:1.0:1775764896.227651:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.227658:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24174, 24174] 00000008:00010000:1.0:1775764896.227662:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.227663:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.227669:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227675:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99016704 99020799) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227680:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227685:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227688:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.227692:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99016704, 99020800) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.227696:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24174 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.227708:0:18302:0:(rw26.c:858:ll_write_end()) pos 99016704, len 4096, copied 4096 00000080:00200000:1.0:1775764896.227711:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000060f0401b[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.227713:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000020892358 17ffffc0001001 3:0 ffff90c212814a80 24174 lru 00000080:00200000:1.0:1775764896.227717:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000043fe8873 24174: 1< 0 - - > 2< 99016704 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1597440 - | + - + - > 5< + - + - | 0 - | 3694 - -> 00000080:00200000:1.0:1775764896.227718:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000060f0401b 00000080:00200000:1.0:1775764896.227719:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.227720:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.227725:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227730:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227734:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227738:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.227745:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.227746:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.227747:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.227754:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.227755:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.227756:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.227759:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227763:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.227769:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.227770:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.227773:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.227776:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24174, 24174] 00000080:00200000:1.0:1775764896.227786:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.227789:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.227816:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99020800 i_blocks 90112 00000080:00200000:1.0:1775764896.227820:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.227836:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99020800, count: 4096 00000080:00200000:1.0:1775764896.227847:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99020800, bytes: 4096 00000080:00200000:1.0:1775764896.227849:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.227853:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24176, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.227855:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99020800, 99024896) 0 0 00000080:00200000:1.0:1775764896.227857:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.227861:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24175, 24175] 00020000:00200000:1.0:1775764896.227863:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99020800 chunk: [99020800, 99024896) 99024896, 4096 00020000:00200000:1.0:1775764896.227864:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.227890:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24175, 24175] 00000008:00010000:1.0:1775764896.227893:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.227895:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.227901:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227906:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99020800 99024895) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227912:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227920:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.227924:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.227929:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99020800, 99024896) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.227932:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24175 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.227945:0:18302:0:(rw26.c:858:ll_write_end()) pos 99020800, len 4096, copied 4096 00000080:00200000:1.0:1775764896.227949:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000e3fe8f6[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.227952:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cc211604 17ffffc0001001 3:0 ffff90c212814700 24175 lru 00000080:00200000:1.0:1775764896.227957:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007cbd73b4 24175: 1< 0 - - > 2< 99020800 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1593344 - | + - + - > 5< + - + - | 0 - | 3695 - -> 00000080:00200000:1.0:1775764896.227959:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000e3fe8f6 00000080:00200000:1.0:1775764896.227960:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.227962:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.227970:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227977:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227983:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.227990:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.228005:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.228009:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.228011:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.228021:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.228023:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.228025:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.228029:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228036:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.228044:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.228047:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.228051:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.228054:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24175, 24175] 00000080:00200000:1.0:1775764896.228064:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.228067:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.228096:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99024896 i_blocks 90112 00000080:00200000:1.0:1775764896.228100:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.228270:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99024896, count: 4096 00000080:00200000:1.0:1775764896.228280:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99024896, bytes: 4096 00000080:00200000:1.0:1775764896.228282:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.228284:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24177, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.228285:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99024896, 99028992) 0 0 00000080:00200000:1.0:1775764896.228287:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.228291:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24176, 24176] 00020000:00200000:1.0:1775764896.228294:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99024896 chunk: [99024896, 99028992) 99028992, 4096 00020000:00200000:1.0:1775764896.228295:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.228302:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24176, 24176] 00000008:00010000:1.0:1775764896.228306:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.228307:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.228313:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228319:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99024896 99028991) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228325:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228332:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228339:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.228346:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99024896, 99028992) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.228352:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24176 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.228370:0:18302:0:(rw26.c:858:ll_write_end()) pos 99024896, len 4096, copied 4096 00000080:00200000:1.0:1775764896.228374:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000807f5807[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.228377:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009ec0fc06 17ffffc0001001 3:0 ffff90c212814c40 24176 lru 00000080:00200000:1.0:1775764896.228383:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005811d737 24176: 1< 0 - - > 2< 99024896 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1589248 - | + - + - > 5< + - + - | 0 - | 3696 - -> 00000080:00200000:1.0:1775764896.228385:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000807f5807 00000080:00200000:1.0:1775764896.228386:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.228388:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.228396:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.228404:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.228411:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.228416:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.228424:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.228426:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.228427:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.228434:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.228436:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.228438:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.228442:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228449:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.228457:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.228460:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.228464:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.228467:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24176, 24176] 00000080:00200000:1.0:1775764896.228478:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.228481:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.228509:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99028992 i_blocks 90112 00000080:00200000:1.0:1775764896.228512:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.228523:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99028992, count: 4096 00000080:00200000:1.0:1775764896.228529:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99028992, bytes: 4096 00000080:00200000:1.0:1775764896.228531:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.228534:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24178, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.228535:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99028992, 99033088) 0 0 00000080:00200000:1.0:1775764896.228536:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.228541:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24177, 24177] 00020000:00200000:1.0:1775764896.228544:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99028992 chunk: [99028992, 99033088) 99033088, 4096 00020000:00200000:1.0:1775764896.228546:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.228557:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24177, 24177] 00000008:00010000:1.0:1775764896.228562:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.228564:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.228572:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228581:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99028992 99033087) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228589:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228597:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228604:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.228612:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99028992, 99033088) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.228616:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24177 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.228628:0:18302:0:(rw26.c:858:ll_write_end()) pos 99028992, len 4096, copied 4096 00000080:00200000:1.0:1775764896.228631:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000c1ed4e5[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.228633:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000036439023 17ffffc0001001 3:0 ffff90c2128148c0 24177 lru 00000080:00200000:1.0:1775764896.228636:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000063fb60e3 24177: 1< 0 - - > 2< 99028992 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1585152 - | + - + - > 5< + - + - | 0 - | 3697 - -> 00000080:00200000:1.0:1775764896.228638:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000c1ed4e5 00000080:00200000:1.0:1775764896.228639:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.228640:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.228645:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.228650:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.228654:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.228658:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.228665:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.228667:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.228668:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.228675:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.228676:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.228678:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.228681:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228686:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.228691:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.228692:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.228695:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.228697:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24177, 24177] 00000080:00200000:1.0:1775764896.228704:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.228706:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.228781:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99033088 i_blocks 90112 00000080:00200000:1.0:1775764896.228785:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.228803:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99033088, count: 4096 00000080:00200000:1.0:1775764896.228814:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99033088, bytes: 4096 00000080:00200000:1.0:1775764896.228816:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.228820:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24179, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.228822:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99033088, 99037184) 0 0 00000080:00200000:1.0:1775764896.228824:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.228829:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24178, 24178] 00020000:00200000:1.0:1775764896.228832:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99033088 chunk: [99033088, 99037184) 99037184, 4096 00020000:00200000:1.0:1775764896.228835:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.228846:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24178, 24178] 00000008:00010000:1.0:1775764896.228851:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.228854:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.228860:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228866:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99033088 99037183) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228888:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228893:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.228897:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.228902:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99033088, 99037184) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.228906:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24178 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.228918:0:18302:0:(rw26.c:858:ll_write_end()) pos 99033088, len 4096, copied 4096 00000080:00200000:1.0:1775764896.228921:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006e40d83a[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.228923:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000511870ee 17ffffc0001001 3:0 ffff90c212814b60 24178 lru 00000080:00200000:1.0:1775764896.228926:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000fcec6e4d 24178: 1< 0 - - > 2< 99033088 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1581056 - | + - + - > 5< + - + - | 0 - | 3698 - -> 00000080:00200000:1.0:1775764896.228927:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006e40d83a 00000080:00200000:1.0:1775764896.228928:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.228930:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.228935:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.228941:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.228948:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.228960:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.228971:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.228974:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.228976:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.228986:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.228988:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.228991:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.228995:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229000:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.229005:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.229007:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.229009:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.229011:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24178, 24178] 00000080:00200000:1.0:1775764896.229018:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.229020:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.229040:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99037184 i_blocks 90112 00000080:00200000:1.0:1775764896.229042:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.229053:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99037184, count: 4096 00000080:00200000:1.0:1775764896.229058:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99037184, bytes: 4096 00000080:00200000:1.0:1775764896.229060:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.229063:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24180, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.229064:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99037184, 99041280) 0 0 00000080:00200000:1.0:1775764896.229066:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.229069:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24179, 24179] 00020000:00200000:1.0:1775764896.229071:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99037184 chunk: [99037184, 99041280) 99041280, 4096 00020000:00200000:1.0:1775764896.229073:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.229079:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24179, 24179] 00000008:00010000:1.0:1775764896.229082:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.229084:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.229089:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229095:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99037184 99041279) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229100:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229105:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229109:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.229123:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99037184, 99041280) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.229128:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24179 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.229146:0:18302:0:(rw26.c:858:ll_write_end()) pos 99037184, len 4096, copied 4096 00000080:00200000:1.0:1775764896.229150:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e07e56b6[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.229153:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a9eb9164 17ffffc0001001 3:0 ffff90c212814000 24179 lru 00000080:00200000:1.0:1775764896.229158:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000789561c 24179: 1< 0 - - > 2< 99037184 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1576960 - | + - + - > 5< + - + - | 0 - | 3699 - -> 00000080:00200000:1.0:1775764896.229161:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e07e56b6 00000080:00200000:1.0:1775764896.229162:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.229164:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.229171:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.229177:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.229184:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.229191:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.229202:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.229205:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.229207:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.229216:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.229219:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.229221:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.229225:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229234:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.229241:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.229244:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.229248:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.229250:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24179, 24179] 00000080:00200000:1.0:1775764896.229261:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.229264:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.229292:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99041280 i_blocks 90112 00000080:00200000:1.0:1775764896.229296:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.229311:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99041280, count: 4096 00000080:00200000:1.0:1775764896.229322:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99041280, bytes: 4096 00000080:00200000:1.0:1775764896.229325:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.229328:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24181, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.229330:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99041280, 99045376) 0 0 00000080:00200000:1.0:1775764896.229332:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.229337:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24180, 24180] 00020000:00200000:1.0:1775764896.229340:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99041280 chunk: [99041280, 99045376) 99045376, 4096 00020000:00200000:1.0:1775764896.229342:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.229352:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24180, 24180] 00000008:00010000:1.0:1775764896.229358:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.229360:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.229368:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229378:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99041280 99045375) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229386:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229393:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229403:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.229411:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99041280, 99045376) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.229416:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24180 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.229431:0:18302:0:(rw26.c:858:ll_write_end()) pos 99041280, len 4096, copied 4096 00000080:00200000:1.0:1775764896.229435:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fb596cac[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.229438:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ff3d3125 17ffffc0001001 3:0 ffff90c212814ee0 24180 lru 00000080:00200000:1.0:1775764896.229444:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000096e62b39 24180: 1< 0 - - > 2< 99041280 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1572864 - | + - + - > 5< + - + - | 0 - | 3700 - -> 00000080:00200000:1.0:1775764896.229446:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fb596cac 00000080:00200000:1.0:1775764896.229447:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.229449:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.229456:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.229464:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.229471:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.229478:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.229489:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.229492:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.229495:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.229504:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.229507:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.229509:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.229513:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229520:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.229528:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.229532:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.229535:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.229538:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24180, 24180] 00000080:00200000:1.0:1775764896.229548:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.229551:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.229580:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99045376 i_blocks 90112 00000080:00200000:1.0:1775764896.229583:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.229599:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99045376, count: 4096 00000080:00200000:1.0:1775764896.229610:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99045376, bytes: 4096 00000080:00200000:1.0:1775764896.229612:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.229616:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24182, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.229617:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99045376, 99049472) 0 0 00000080:00200000:1.0:1775764896.229619:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.229625:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24181, 24181] 00020000:00200000:1.0:1775764896.229627:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99045376 chunk: [99045376, 99049472) 99049472, 4096 00020000:00200000:1.0:1775764896.229630:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.229642:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24181, 24181] 00000008:00010000:1.0:1775764896.229647:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.229650:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.229658:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229666:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99045376 99049471) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229675:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229682:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229690:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.229697:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99045376, 99049472) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.229703:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24181 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.229718:0:18302:0:(rw26.c:858:ll_write_end()) pos 99045376, len 4096, copied 4096 00000080:00200000:1.0:1775764896.229722:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002c6e540d[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.229725:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000bc221d06 17ffffc0001001 3:0 ffff90c2128140e0 24181 lru 00000080:00200000:1.0:1775764896.229811:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000043e7128f 24181: 1< 0 - - > 2< 99045376 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1568768 - | + - + - > 5< + - + - | 0 - | 3701 - -> 00000080:00200000:1.0:1775764896.229814:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002c6e540d 00000080:00200000:1.0:1775764896.229815:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.229818:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.229827:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.229835:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.229843:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.229851:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.229861:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.229864:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.229866:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.229898:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.229901:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.229904:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.229908:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.229916:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.229924:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.229926:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.229930:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.229933:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24181, 24181] 00000080:00200000:1.0:1775764896.229944:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.229947:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.229976:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99049472 i_blocks 90112 00000080:00200000:1.0:1775764896.229980:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.229996:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99049472, count: 4096 00000080:00200000:1.0:1775764896.230006:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99049472, bytes: 4096 00000080:00200000:1.0:1775764896.230009:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.230012:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24183, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.230014:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99049472, 99053568) 0 0 00000080:00200000:1.0:1775764896.230016:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.230021:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24182, 24182] 00020000:00200000:1.0:1775764896.230024:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99049472 chunk: [99049472, 99053568) 99053568, 4096 00020000:00200000:1.0:1775764896.230026:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.230036:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24182, 24182] 00000008:00010000:1.0:1775764896.230041:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.230044:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.230051:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230060:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99049472 99053567) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230068:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230075:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230081:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.230089:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99049472, 99053568) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.230094:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24182 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.230111:0:18302:0:(rw26.c:858:ll_write_end()) pos 99049472, len 4096, copied 4096 00000080:00200000:1.0:1775764896.230125:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000056bb4aa6[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.230128:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006a8faafc 17ffffc0001001 3:0 ffff90c2128141c0 24182 lru 00000080:00200000:1.0:1775764896.230133:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000060d8c164 24182: 1< 0 - - > 2< 99049472 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1564672 - | + - + - > 5< + - + - | 0 - | 3702 - -> 00000080:00200000:1.0:1775764896.230135:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000056bb4aa6 00000080:00200000:1.0:1775764896.230136:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.230139:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.230147:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.230154:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.230160:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.230167:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.230179:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.230182:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.230184:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.230193:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.230196:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.230199:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.230203:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230210:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.230218:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.230220:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.230224:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.230227:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24182, 24182] 00000080:00200000:1.0:1775764896.230237:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.230241:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.230269:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99053568 i_blocks 90112 00000080:00200000:1.0:1775764896.230272:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.230283:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99053568, count: 4096 00000080:00200000:1.0:1775764896.230290:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99053568, bytes: 4096 00000080:00200000:1.0:1775764896.230291:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.230294:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24184, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.230295:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99053568, 99057664) 0 0 00000080:00200000:1.0:1775764896.230297:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.230301:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24183, 24183] 00020000:00200000:1.0:1775764896.230303:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99053568 chunk: [99053568, 99057664) 99057664, 4096 00020000:00200000:1.0:1775764896.230304:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.230311:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24183, 24183] 00000008:00010000:1.0:1775764896.230315:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.230316:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.230322:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230330:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99053568 99057663) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230338:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230351:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230358:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.230367:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99053568, 99057664) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.230372:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24183 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.230395:0:18302:0:(rw26.c:858:ll_write_end()) pos 99053568, len 4096, copied 4096 00000080:00200000:1.0:1775764896.230399:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009c85c8a6[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.230401:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c184dfcf 17ffffc0001021 2:0 ffff90c2128147e0 24183 lru 00000080:00200000:1.0:1775764896.230404:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c07378c5 24183: 1< 0 - - > 2< 99053568 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1560576 - | + - + - > 5< + - + - | 0 - | 3703 - -> 00000080:00200000:1.0:1775764896.230406:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009c85c8a6 00000080:00200000:1.0:1775764896.230407:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.230408:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.230414:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.230419:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.230423:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.230427:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.230440:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.230443:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.230445:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.230454:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.230457:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.230459:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.230463:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230471:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.230479:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.230482:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.230486:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.230489:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24183, 24183] 00000080:00200000:1.0:1775764896.230499:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.230502:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.230531:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99057664 i_blocks 90112 00000080:00200000:1.0:1775764896.230534:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.230551:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99057664, count: 4096 00000080:00200000:1.0:1775764896.230561:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99057664, bytes: 4096 00000080:00200000:1.0:1775764896.230565:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.230568:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24185, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.230570:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99057664, 99061760) 0 0 00000080:00200000:1.0:1775764896.230572:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.230576:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24184, 24184] 00020000:00200000:1.0:1775764896.230579:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99057664 chunk: [99057664, 99061760) 99061760, 4096 00020000:00200000:1.0:1775764896.230582:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.230593:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24184, 24184] 00000008:00010000:1.0:1775764896.230599:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.230601:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.230609:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230618:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99057664 99061759) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230627:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230634:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230640:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.230647:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99057664, 99061760) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.230653:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24184 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.230668:0:18302:0:(rw26.c:858:ll_write_end()) pos 99057664, len 4096, copied 4096 00000080:00200000:1.0:1775764896.230672:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d0e789e2[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.230675:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b8fd8db3 17ffffc0001001 3:0 ffff90c2128149a0 24184 lru 00000080:00200000:1.0:1775764896.230680:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008282f78d 24184: 1< 0 - - > 2< 99057664 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1556480 - | + - + - > 5< + - + - | 0 - | 3704 - -> 00000080:00200000:1.0:1775764896.230683:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d0e789e2 00000080:00200000:1.0:1775764896.230684:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.230686:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.230694:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.230701:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.230707:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.230714:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.230725:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.230815:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.230819:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.230831:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.230833:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.230835:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.230840:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.230848:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.230856:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.230858:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.230863:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.230865:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24184, 24184] 00000080:00200000:1.0:1775764896.230968:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.230972:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.231004:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99061760 i_blocks 90112 00000080:00200000:1.0:1775764896.231008:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.231025:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99061760, count: 4096 00000080:00200000:1.0:1775764896.231037:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99061760, bytes: 4096 00000080:00200000:1.0:1775764896.231039:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.231043:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24186, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.231045:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99061760, 99065856) 0 0 00000080:00200000:1.0:1775764896.231046:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.231051:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24185, 24185] 00020000:00200000:1.0:1775764896.231053:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99061760 chunk: [99061760, 99065856) 99065856, 4096 00020000:00200000:1.0:1775764896.231054:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.231062:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24185, 24185] 00000008:00010000:1.0:1775764896.231066:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.231068:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.231074:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231079:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99061760 99065855) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231085:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231089:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231093:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.231099:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99061760, 99065856) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.231102:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24185 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.231128:0:18302:0:(rw26.c:858:ll_write_end()) pos 99061760, len 4096, copied 4096 00000080:00200000:1.0:1775764896.231132:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ec2964e9[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.231134:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000015d5163b 17ffffc0001001 3:0 ffff90c2128142a0 24185 lru 00000080:00200000:1.0:1775764896.231137:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b6e1613f 24185: 1< 0 - - > 2< 99061760 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1552384 - | + - + - > 5< + - + - | 0 - | 3705 - -> 00000080:00200000:1.0:1775764896.231139:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ec2964e9 00000080:00200000:1.0:1775764896.231139:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.231141:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.231147:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.231152:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.231156:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.231160:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.231167:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.231169:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.231170:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.231177:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.231179:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.231180:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.231183:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231188:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.231194:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.231195:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.231198:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.231200:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24185, 24185] 00000080:00200000:1.0:1775764896.231207:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.231209:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.231228:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99065856 i_blocks 90112 00000080:00200000:1.0:1775764896.231230:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.231241:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99065856, count: 4096 00000080:00200000:1.0:1775764896.231246:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99065856, bytes: 4096 00000080:00200000:1.0:1775764896.231248:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.231251:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24187, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.231252:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99065856, 99069952) 0 0 00000080:00200000:1.0:1775764896.231253:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.231257:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24186, 24186] 00020000:00200000:1.0:1775764896.231259:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99065856 chunk: [99065856, 99069952) 99069952, 4096 00020000:00200000:1.0:1775764896.231260:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.231266:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24186, 24186] 00000008:00010000:1.0:1775764896.231270:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.231271:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.231276:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231282:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99065856 99069951) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231288:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231293:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231296:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.231301:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99065856, 99069952) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.231304:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24186 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.231315:0:18302:0:(rw26.c:858:ll_write_end()) pos 99065856, len 4096, copied 4096 00000080:00200000:1.0:1775764896.231318:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f1b96696[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.231320:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c62d8bfa 17ffffc0001001 3:0 ffff90c212814d20 24186 lru 00000080:00200000:1.0:1775764896.231323:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a4af56e0 24186: 1< 0 - - > 2< 99065856 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1548288 - | + - + - > 5< + - + - | 0 - | 3706 - -> 00000080:00200000:1.0:1775764896.231325:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f1b96696 00000080:00200000:1.0:1775764896.231326:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.231328:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.231335:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.231342:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.231349:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.231360:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.231371:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.231374:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.231376:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.231385:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.231388:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.231390:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.231394:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231402:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.231410:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.231412:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.231415:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.231417:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24186, 24186] 00000080:00200000:1.0:1775764896.231424:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.231426:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.231450:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99069952 i_blocks 90112 00000080:00200000:1.0:1775764896.231453:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.231469:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99069952, count: 4096 00000080:00200000:1.0:1775764896.231479:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99069952, bytes: 4096 00000080:00200000:1.0:1775764896.231482:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.231486:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24188, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.231488:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99069952, 99074048) 0 0 00000080:00200000:1.0:1775764896.231490:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.231495:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24187, 24187] 00020000:00200000:1.0:1775764896.231498:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99069952 chunk: [99069952, 99074048) 99074048, 4096 00020000:00200000:1.0:1775764896.231501:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.231512:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24187, 24187] 00000008:00010000:1.0:1775764896.231516:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.231519:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.231527:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231536:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99069952 99074047) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231544:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231552:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231559:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.231567:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99069952, 99074048) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.231572:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24187 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.231588:0:18302:0:(rw26.c:858:ll_write_end()) pos 99069952, len 4096, copied 4096 00000080:00200000:1.0:1775764896.231592:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006942221b[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.231595:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d2b2d176 17ffffc0001001 3:0 ffff90c212814380 24187 lru 00000080:00200000:1.0:1775764896.231601:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c62ec80a 24187: 1< 0 - - > 2< 99069952 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1544192 - | + - + - > 5< + - + - | 0 - | 3707 - -> 00000080:00200000:1.0:1775764896.231603:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006942221b 00000080:00200000:1.0:1775764896.231604:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.231606:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.231614:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.231623:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.231629:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.231636:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.231647:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.231650:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.231653:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.231662:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.231665:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.231667:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.231671:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231679:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.231687:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.231689:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.231693:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.231696:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24187, 24187] 00000080:00200000:1.0:1775764896.231706:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.231709:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.231737:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99074048 i_blocks 90112 00000080:00200000:1.0:1775764896.231741:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.231756:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99074048, count: 4096 00000080:00200000:1.0:1775764896.231766:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99074048, bytes: 4096 00000080:00200000:1.0:1775764896.231769:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.231773:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24189, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.231774:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99074048, 99078144) 0 0 00000080:00200000:1.0:1775764896.231777:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.231782:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24188, 24188] 00020000:00200000:1.0:1775764896.231785:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99074048 chunk: [99074048, 99078144) 99078144, 4096 00020000:00200000:1.0:1775764896.231787:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.231799:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24188, 24188] 00000008:00010000:1.0:1775764896.231804:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.231807:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.231814:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231822:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99074048 99078143) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231830:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231837:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.231849:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.231857:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99074048, 99078144) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.231862:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24188 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.231967:0:18302:0:(rw26.c:858:ll_write_end()) pos 99074048, len 4096, copied 4096 00000080:00200000:1.0:1775764896.231972:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006befe865[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.231980:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000049ada20d 17ffffc0001001 3:0 ffff90c212814620 24188 lru 00000080:00200000:1.0:1775764896.231986:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000eaca5b21 24188: 1< 0 - - > 2< 99074048 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1540096 - | + - + - > 5< + - + - | 0 - | 3708 - -> 00000080:00200000:1.0:1775764896.231989:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006befe865 00000080:00200000:1.0:1775764896.231990:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.231992:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.232000:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232008:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232015:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232022:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.232034:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.232037:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.232041:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.232050:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.232053:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.232056:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.232059:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232067:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.232075:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.232077:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.232081:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.232084:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24188, 24188] 00000080:00200000:1.0:1775764896.232094:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.232097:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.232138:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99078144 i_blocks 90112 00000080:00200000:1.0:1775764896.232142:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.232159:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99078144, count: 4096 00000080:00200000:1.0:1775764896.232169:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99078144, bytes: 4096 00000080:00200000:1.0:1775764896.232172:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.232175:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24190, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.232176:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99078144, 99082240) 0 0 00000080:00200000:1.0:1775764896.232178:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.232184:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24189, 24189] 00020000:00200000:1.0:1775764896.232188:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99078144 chunk: [99078144, 99082240) 99082240, 4096 00020000:00200000:1.0:1775764896.232190:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.232201:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24189, 24189] 00000008:00010000:1.0:1775764896.232206:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.232209:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.232217:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232225:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99078144 99082239) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232233:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232239:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232245:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.232252:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99078144, 99082240) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.232257:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24189 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.232274:0:18302:0:(rw26.c:858:ll_write_end()) pos 99078144, len 4096, copied 4096 00000080:00200000:1.0:1775764896.232278:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001a6e2ae7[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.232281:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a9375309 17ffffc0001001 3:0 ffff90c212814e00 24189 lru 00000080:00200000:1.0:1775764896.232286:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f1c00f83 24189: 1< 0 - - > 2< 99078144 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1536000 - | + - + - > 5< + - + - | 0 - | 3709 - -> 00000080:00200000:1.0:1775764896.232288:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001a6e2ae7 00000080:00200000:1.0:1775764896.232289:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.232291:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.232299:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232307:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232314:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232321:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.232332:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.232335:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.232337:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.232346:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.232349:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.232351:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.232355:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232363:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.232371:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.232373:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.232377:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.232379:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24189, 24189] 00000080:00200000:1.0:1775764896.232390:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.232393:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.232421:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99082240 i_blocks 90112 00000080:00200000:1.0:1775764896.232424:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.232440:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99082240, count: 4096 00000080:00200000:1.0:1775764896.232451:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99082240, bytes: 4096 00000080:00200000:1.0:1775764896.232454:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.232457:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24191, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.232459:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99082240, 99086336) 0 0 00000080:00200000:1.0:1775764896.232461:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.232466:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24190, 24190] 00020000:00200000:1.0:1775764896.232469:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99082240 chunk: [99082240, 99086336) 99086336, 4096 00020000:00200000:1.0:1775764896.232471:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.232483:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24190, 24190] 00000008:00010000:1.0:1775764896.232488:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.232490:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.232498:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232507:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99082240 99086335) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232515:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232522:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232529:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.232537:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99082240, 99086336) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.232542:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24190 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.232570:0:18302:0:(rw26.c:858:ll_write_end()) pos 99082240, len 4096, copied 4096 00000080:00200000:1.0:1775764896.232575:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000000b12c1f[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.232577:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000348ee792 17ffffc0001001 3:0 ffff90c2128157e0 24190 lru 00000080:00200000:1.0:1775764896.232583:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d9ce0ee4 24190: 1< 0 - - > 2< 99082240 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1531904 - | + - + - > 5< + - + - | 0 - | 3710 - -> 00000080:00200000:1.0:1775764896.232585:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000000b12c1f 00000080:00200000:1.0:1775764896.232586:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.232588:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.232596:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232605:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232612:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232619:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.232631:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.232634:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.232636:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.232646:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.232648:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.232650:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.232654:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232662:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.232670:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.232673:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.232677:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.232680:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24190, 24190] 00000080:00200000:1.0:1775764896.232690:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.232693:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.232721:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99086336 i_blocks 90112 00000080:00200000:1.0:1775764896.232724:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.232739:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99086336, count: 4096 00000080:00200000:1.0:1775764896.232750:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99086336, bytes: 4096 00000080:00200000:1.0:1775764896.232752:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.232756:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24192, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.232758:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99086336, 99090432) 0 0 00000080:00200000:1.0:1775764896.232760:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.232765:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24191, 24191] 00020000:00200000:1.0:1775764896.232768:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99086336 chunk: [99086336, 99090432) 99090432, 4096 00020000:00200000:1.0:1775764896.232770:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.232782:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24191, 24191] 00000008:00010000:1.0:1775764896.232787:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.232789:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.232797:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232806:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99086336 99090431) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232814:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232827:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232834:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.232842:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99086336, 99090432) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.232847:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24191 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.232864:0:18302:0:(rw26.c:858:ll_write_end()) pos 99086336, len 4096, copied 4096 00000080:00200000:1.0:1775764896.232867:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007462ccb1[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.232893:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000072f2cba1 17ffffc0001001 3:0 ffff90c2128159a0 24191 lru 00000080:00200000:1.0:1775764896.232899:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000985a0a3a 24191: 1< 0 - - > 2< 99086336 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1527808 - | + - + - > 5< + - + - | 0 - | 3711 - -> 00000080:00200000:1.0:1775764896.232901:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007462ccb1 00000080:00200000:1.0:1775764896.232902:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.232904:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.232913:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232920:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232927:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.232933:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.232950:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.232953:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.232955:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.232965:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.232968:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.232970:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.232974:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.232982:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.232989:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.232992:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.232996:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.232998:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24191, 24191] 00000080:00200000:1.0:1775764896.233009:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.233012:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.233040:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99090432 i_blocks 90112 00000080:00200000:1.0:1775764896.233043:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.233057:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99090432, count: 4096 00000080:00200000:1.0:1775764896.233066:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99090432, bytes: 4096 00000080:00200000:1.0:1775764896.233069:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.233073:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24193, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.233075:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99090432, 99094528) 0 0 00000080:00200000:1.0:1775764896.233077:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.233082:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24192, 24192] 00020000:00200000:1.0:1775764896.233085:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99090432 chunk: [99090432, 99094528) 99094528, 4096 00020000:00200000:1.0:1775764896.233087:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.233099:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24192, 24192] 00000008:00010000:1.0:1775764896.233104:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.233107:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.233126:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233136:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99090432 99094527) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233144:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233151:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233159:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.233166:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99090432, 99094528) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.233171:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24192 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.233189:0:18302:0:(rw26.c:858:ll_write_end()) pos 99090432, len 4096, copied 4096 00000080:00200000:1.0:1775764896.233192:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005acdea1a[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.233194:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cf167a6c 17ffffc0001001 3:0 ffff90c2128152a0 24192 lru 00000080:00200000:1.0:1775764896.233197:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005b0a95c6 24192: 1< 0 - - > 2< 99090432 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1523712 - | + - + - > 5< + - + - | 0 - | 3712 - -> 00000080:00200000:1.0:1775764896.233200:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005acdea1a 00000080:00200000:1.0:1775764896.233200:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.233202:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.233207:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.233212:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.233217:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.233221:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.233228:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.233229:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.233231:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.233238:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.233239:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.233240:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.233243:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233247:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.233252:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.233254:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.233256:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.233258:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24192, 24192] 00000080:00200000:1.0:1775764896.233266:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.233268:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.233287:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99094528 i_blocks 90112 00000080:00200000:1.0:1775764896.233289:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.233300:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99094528, count: 4096 00000080:00200000:1.0:1775764896.233306:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99094528, bytes: 4096 00000080:00200000:1.0:1775764896.233308:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.233310:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24194, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.233311:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99094528, 99098624) 0 0 00000080:00200000:1.0:1775764896.233313:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.233317:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24193, 24193] 00020000:00200000:1.0:1775764896.233318:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99094528 chunk: [99094528, 99098624) 99098624, 4096 00020000:00200000:1.0:1775764896.233320:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.233327:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24193, 24193] 00000008:00010000:1.0:1775764896.233331:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.233334:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.233341:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233349:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99094528 99098623) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233357:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233364:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233370:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.233379:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99094528, 99098624) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.233383:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24193 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.233399:0:18302:0:(rw26.c:858:ll_write_end()) pos 99094528, len 4096, copied 4096 00000080:00200000:1.0:1775764896.233403:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e0e2a8d8[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.233406:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000835a1763 17ffffc0001001 3:0 ffff90c212815d20 24193 lru 00000080:00200000:1.0:1775764896.233411:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a14fb8c9 24193: 1< 0 - - > 2< 99094528 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1519616 - | + - + - > 5< + - + - | 0 - | 3713 - -> 00000080:00200000:1.0:1775764896.233413:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e0e2a8d8 00000080:00200000:1.0:1775764896.233414:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.233415:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.233421:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.233426:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.233430:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.233434:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.233442:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.233445:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.233447:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.233455:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.233457:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.233460:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.233464:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233472:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.233480:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.233482:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.233486:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.233489:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24193, 24193] 00000080:00200000:1.0:1775764896.233499:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.233503:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.233531:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99098624 i_blocks 90112 00000080:00200000:1.0:1775764896.233535:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.233551:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99098624, count: 4096 00000080:00200000:1.0:1775764896.233561:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99098624, bytes: 4096 00000080:00200000:1.0:1775764896.233564:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.233567:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24195, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.233568:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99098624, 99102720) 0 0 00000080:00200000:1.0:1775764896.233571:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.233576:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24194, 24194] 00020000:00200000:1.0:1775764896.233578:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99098624 chunk: [99098624, 99102720) 99102720, 4096 00020000:00200000:1.0:1775764896.233580:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.233591:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24194, 24194] 00000008:00010000:1.0:1775764896.233597:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.233599:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.233607:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233615:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99098624 99102719) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233624:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233631:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233639:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.233647:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99098624, 99102720) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.233651:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24194 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.233667:0:18302:0:(rw26.c:858:ll_write_end()) pos 99098624, len 4096, copied 4096 00000080:00200000:1.0:1775764896.233672:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008e853a10[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.233675:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000042a04b6e 17ffffc0001001 3:0 ffff90c212815380 24194 lru 00000080:00200000:1.0:1775764896.233680:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000041c9811b 24194: 1< 0 - - > 2< 99098624 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1515520 - | + - + - > 5< + - + - | 0 - | 3714 - -> 00000080:00200000:1.0:1775764896.233682:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008e853a10 00000080:00200000:1.0:1775764896.233683:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.233685:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.233693:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.233700:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.233707:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.233721:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.233732:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.233735:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.233737:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.233747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.233750:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.233752:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.233756:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.233764:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.233772:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.233775:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.233779:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.233781:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24194, 24194] 00000080:00200000:1.0:1775764896.233792:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.233795:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.233823:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99102720 i_blocks 90112 00000080:00200000:1.0:1775764896.233826:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.234041:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99102720, count: 4096 00000080:00200000:1.0:1775764896.234051:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99102720, bytes: 4096 00000080:00200000:1.0:1775764896.234052:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.234056:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24196, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.234057:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99102720, 99106816) 0 0 00000080:00200000:1.0:1775764896.234058:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.234063:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24195, 24195] 00020000:00200000:1.0:1775764896.234065:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99102720 chunk: [99102720, 99106816) 99106816, 4096 00020000:00200000:1.0:1775764896.234066:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.234074:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24195, 24195] 00000008:00010000:1.0:1775764896.234077:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.234079:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.234085:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234090:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99102720 99106815) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234096:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234101:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234104:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.234109:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99102720, 99106816) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.234124:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24195 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.234138:0:18302:0:(rw26.c:858:ll_write_end()) pos 99102720, len 4096, copied 4096 00000080:00200000:1.0:1775764896.234141:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a5b27c34[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.234143:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b35a3014 17ffffc0001001 3:0 ffff90c212815620 24195 lru 00000080:00200000:1.0:1775764896.234147:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d6e33891 24195: 1< 0 - - > 2< 99102720 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1511424 - | + - + - > 5< + - + - | 0 - | 3715 - -> 00000080:00200000:1.0:1775764896.234149:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a5b27c34 00000080:00200000:1.0:1775764896.234150:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.234151:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.234157:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234162:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234166:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234170:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.234177:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.234179:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.234180:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.234187:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.234189:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.234190:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.234193:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234198:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.234203:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.234205:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.234208:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.234210:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24195, 24195] 00000080:00200000:1.0:1775764896.234217:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.234219:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.234238:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99106816 i_blocks 90112 00000080:00200000:1.0:1775764896.234240:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.234250:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99106816, count: 4096 00000080:00200000:1.0:1775764896.234257:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99106816, bytes: 4096 00000080:00200000:1.0:1775764896.234258:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.234261:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24197, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.234262:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99106816, 99110912) 0 0 00000080:00200000:1.0:1775764896.234263:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.234267:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24196, 24196] 00020000:00200000:1.0:1775764896.234269:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99106816 chunk: [99106816, 99110912) 99110912, 4096 00020000:00200000:1.0:1775764896.234270:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.234276:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24196, 24196] 00000008:00010000:1.0:1775764896.234280:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.234281:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.234286:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234292:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99106816 99110911) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234298:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234302:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234310:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.234314:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99106816, 99110912) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.234318:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24196 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.234330:0:18302:0:(rw26.c:858:ll_write_end()) pos 99106816, len 4096, copied 4096 00000080:00200000:1.0:1775764896.234335:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c6cf4c0d[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.234337:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004dd704f9 17ffffc0001001 3:0 ffff90c212815e00 24196 lru 00000080:00200000:1.0:1775764896.234342:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b1242d1f 24196: 1< 0 - - > 2< 99106816 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1507328 - | + - + - > 5< + - + - | 0 - | 3716 - -> 00000080:00200000:1.0:1775764896.234344:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c6cf4c0d 00000080:00200000:1.0:1775764896.234345:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.234347:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.234355:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234362:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234369:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234376:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.234387:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.234390:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.234393:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.234402:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.234405:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.234407:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.234410:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234415:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.234421:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.234422:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.234425:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.234427:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24196, 24196] 00000080:00200000:1.0:1775764896.234434:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.234436:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.234463:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99110912 i_blocks 90112 00000080:00200000:1.0:1775764896.234467:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.234482:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99110912, count: 4096 00000080:00200000:1.0:1775764896.234493:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99110912, bytes: 4096 00000080:00200000:1.0:1775764896.234496:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.234499:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24198, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.234501:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99110912, 99115008) 0 0 00000080:00200000:1.0:1775764896.234503:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.234508:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24197, 24197] 00020000:00200000:1.0:1775764896.234511:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99110912 chunk: [99110912, 99115008) 99115008, 4096 00020000:00200000:1.0:1775764896.234513:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.234525:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24197, 24197] 00000008:00010000:1.0:1775764896.234530:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.234532:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.234540:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234549:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99110912 99115007) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234558:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234565:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234572:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.234579:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99110912, 99115008) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.234584:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24197 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.234602:0:18302:0:(rw26.c:858:ll_write_end()) pos 99110912, len 4096, copied 4096 00000080:00200000:1.0:1775764896.234606:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000062ca6ba0[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.234609:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003cc5ac86 17ffffc0001001 3:0 ffff90c212815540 24197 lru 00000080:00200000:1.0:1775764896.234614:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000090bac82 24197: 1< 0 - - > 2< 99110912 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1503232 - | + - + - > 5< + - + - | 0 - | 3717 - -> 00000080:00200000:1.0:1775764896.234616:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000062ca6ba0 00000080:00200000:1.0:1775764896.234617:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.234619:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.234627:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234634:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234641:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234649:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.234661:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.234663:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.234665:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.234675:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.234677:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.234680:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.234683:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234691:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.234699:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.234701:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.234705:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.234708:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24197, 24197] 00000080:00200000:1.0:1775764896.234718:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.234722:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.234750:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99115008 i_blocks 90112 00000080:00200000:1.0:1775764896.234753:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.234769:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99115008, count: 4096 00000080:00200000:1.0:1775764896.234780:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99115008, bytes: 4096 00000080:00200000:1.0:1775764896.234783:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.234786:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24199, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.234787:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99115008, 99119104) 0 0 00000080:00200000:1.0:1775764896.234790:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.234795:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24198, 24198] 00020000:00200000:1.0:1775764896.234798:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99115008 chunk: [99115008, 99119104) 99119104, 4096 00020000:00200000:1.0:1775764896.234800:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.234812:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24198, 24198] 00000008:00010000:1.0:1775764896.234817:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.234819:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.234827:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234836:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99115008 99119103) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234844:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234851:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234858:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.234866:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99115008, 99119104) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.234891:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24198 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.234914:0:18302:0:(rw26.c:858:ll_write_end()) pos 99115008, len 4096, copied 4096 00000080:00200000:1.0:1775764896.234917:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008b183f6e[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.234919:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f8f42523 17ffffc0001021 2:0 ffff90c212815460 24198 lru 00000080:00200000:1.0:1775764896.234923:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008b4676ca 24198: 1< 0 - - > 2< 99115008 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1499136 - | + - + - > 5< + - + - | 0 - | 3718 - -> 00000080:00200000:1.0:1775764896.234924:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008b183f6e 00000080:00200000:1.0:1775764896.234925:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.234926:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.234932:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234937:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234942:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.234949:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.234960:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.234963:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.234965:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.234974:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.234976:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.234979:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.234983:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.234990:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.234999:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.235001:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.235006:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.235008:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24198, 24198] 00000080:00200000:1.0:1775764896.235020:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.235023:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.235052:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99119104 i_blocks 90112 00000080:00200000:1.0:1775764896.235055:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.235067:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99119104, count: 4096 00000080:00200000:1.0:1775764896.235074:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99119104, bytes: 4096 00000080:00200000:1.0:1775764896.235076:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.235078:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24200, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.235079:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99119104, 99123200) 0 0 00000080:00200000:1.0:1775764896.235081:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.235085:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24199, 24199] 00020000:00200000:1.0:1775764896.235087:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99119104 chunk: [99119104, 99123200) 99123200, 4096 00020000:00200000:1.0:1775764896.235088:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.235095:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24199, 24199] 00000008:00010000:1.0:1775764896.235098:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.235100:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.235106:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235111:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99119104 99123199) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235130:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235139:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235143:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.235148:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99119104, 99123200) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.235151:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24199 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.235163:0:18302:0:(rw26.c:858:ll_write_end()) pos 99119104, len 4096, copied 4096 00000080:00200000:1.0:1775764896.235166:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009e700a27[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.235167:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000025213944 17ffffc0001001 3:0 ffff90c212815a80 24199 lru 00000080:00200000:1.0:1775764896.235171:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e2e7036b 24199: 1< 0 - - > 2< 99119104 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1495040 - | + - + - > 5< + - + - | 0 - | 3719 - -> 00000080:00200000:1.0:1775764896.235172:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009e700a27 00000080:00200000:1.0:1775764896.235173:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.235174:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.235179:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235184:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235188:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235192:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.235203:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.235205:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.235206:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.235213:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.235214:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.235216:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.235219:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235224:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.235229:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.235230:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.235233:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.235234:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24199, 24199] 00000080:00200000:1.0:1775764896.235241:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.235243:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.235262:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99123200 i_blocks 90112 00000080:00200000:1.0:1775764896.235264:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.235274:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99123200, count: 4096 00000080:00200000:1.0:1775764896.235280:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99123200, bytes: 4096 00000080:00200000:1.0:1775764896.235282:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.235284:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24201, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.235286:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99123200, 99127296) 0 0 00000080:00200000:1.0:1775764896.235287:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.235291:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24200, 24200] 00020000:00200000:1.0:1775764896.235293:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99123200 chunk: [99123200, 99127296) 99127296, 4096 00020000:00200000:1.0:1775764896.235294:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.235301:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24200, 24200] 00000008:00010000:1.0:1775764896.235304:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.235305:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.235310:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235316:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99123200 99127295) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235321:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235326:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235332:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.235339:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99123200, 99127296) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.235344:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24200 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.235361:0:18302:0:(rw26.c:858:ll_write_end()) pos 99123200, len 4096, copied 4096 00000080:00200000:1.0:1775764896.235365:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f5c8ba91[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.235367:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f19d72c0 17ffffc0001001 3:0 ffff90c212815700 24200 lru 00000080:00200000:1.0:1775764896.235373:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000833b3333 24200: 1< 0 - - > 2< 99123200 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1490944 - | + - + - > 5< + - + - | 0 - | 3720 - -> 00000080:00200000:1.0:1775764896.235375:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f5c8ba91 00000080:00200000:1.0:1775764896.235376:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.235378:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.235387:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235395:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235402:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235409:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.235417:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.235419:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.235421:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.235427:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.235429:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.235431:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.235433:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235439:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.235447:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.235450:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.235453:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.235456:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24200, 24200] 00000080:00200000:1.0:1775764896.235466:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.235469:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.235498:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99127296 i_blocks 90112 00000080:00200000:1.0:1775764896.235502:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.235517:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99127296, count: 4096 00000080:00200000:1.0:1775764896.235527:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99127296, bytes: 4096 00000080:00200000:1.0:1775764896.235530:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.235533:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24202, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.235535:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99127296, 99131392) 0 0 00000080:00200000:1.0:1775764896.235537:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.235542:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24201, 24201] 00020000:00200000:1.0:1775764896.235545:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99127296 chunk: [99127296, 99131392) 99131392, 4096 00020000:00200000:1.0:1775764896.235547:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.235558:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24201, 24201] 00000008:00010000:1.0:1775764896.235563:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.235565:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.235573:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235582:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99127296 99131391) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235590:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235597:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235604:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.235612:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99127296, 99131392) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.235617:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24201 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.235633:0:18302:0:(rw26.c:858:ll_write_end()) pos 99127296, len 4096, copied 4096 00000080:00200000:1.0:1775764896.235637:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000864c81b7[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.235640:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007dc78124 17ffffc0001001 3:0 ffff90c212815c40 24201 lru 00000080:00200000:1.0:1775764896.235646:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000533ecb4c 24201: 1< 0 - - > 2< 99127296 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1486848 - | + - + - > 5< + - + - | 0 - | 3721 - -> 00000080:00200000:1.0:1775764896.235647:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000864c81b7 00000080:00200000:1.0:1775764896.235649:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.235651:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.235658:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235666:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235672:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235680:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.235691:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.235694:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.235696:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.235705:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.235708:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.235710:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.235714:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235722:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.235731:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.235733:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.235738:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.235741:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24201, 24201] 00000080:00200000:1.0:1775764896.235751:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.235754:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.235784:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99131392 i_blocks 90112 00000080:00200000:1.0:1775764896.235787:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.235803:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99131392, count: 4096 00000080:00200000:1.0:1775764896.235814:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99131392, bytes: 4096 00000080:00200000:1.0:1775764896.235816:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.235820:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24203, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.235822:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99131392, 99135488) 0 0 00000080:00200000:1.0:1775764896.235824:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.235829:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24202, 24202] 00020000:00200000:1.0:1775764896.235832:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99131392 chunk: [99131392, 99135488) 99135488, 4096 00020000:00200000:1.0:1775764896.235834:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.235845:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24202, 24202] 00000008:00010000:1.0:1775764896.235851:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.235853:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.235860:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235891:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99131392 99135487) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235901:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235907:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.235914:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.235922:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99131392, 99135488) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.235928:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24202 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.235944:0:18302:0:(rw26.c:858:ll_write_end()) pos 99131392, len 4096, copied 4096 00000080:00200000:1.0:1775764896.235949:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e96c943a[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.235952:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005815eb46 17ffffc0001001 3:0 ffff90c2128158c0 24202 lru 00000080:00200000:1.0:1775764896.235957:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000327ff251 24202: 1< 0 - - > 2< 99131392 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1482752 - | + - + - > 5< + - + - | 0 - | 3722 - -> 00000080:00200000:1.0:1775764896.235959:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e96c943a 00000080:00200000:1.0:1775764896.235960:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.235962:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.235971:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235978:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235985:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.235998:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.236009:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.236012:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.236014:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.236024:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.236027:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.236029:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.236032:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236039:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.236047:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.236049:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.236053:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.236056:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24202, 24202] 00000080:00200000:1.0:1775764896.236067:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.236070:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.236099:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99135488 i_blocks 90112 00000080:00200000:1.0:1775764896.236103:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.236129:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99135488, count: 4096 00000080:00200000:1.0:1775764896.236141:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99135488, bytes: 4096 00000080:00200000:1.0:1775764896.236143:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.236148:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24204, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.236149:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99135488, 99139584) 0 0 00000080:00200000:1.0:1775764896.236152:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.236157:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24203, 24203] 00020000:00200000:1.0:1775764896.236160:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99135488 chunk: [99135488, 99139584) 99139584, 4096 00020000:00200000:1.0:1775764896.236163:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.236174:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24203, 24203] 00000008:00010000:1.0:1775764896.236178:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.236181:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.236188:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236196:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99135488 99139583) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236204:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236210:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236217:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.236224:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99135488, 99139584) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.236229:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24203 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.236246:0:18302:0:(rw26.c:858:ll_write_end()) pos 99135488, len 4096, copied 4096 00000080:00200000:1.0:1775764896.236250:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c436d563[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.236252:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000463bf3ae 17ffffc0001001 3:0 ffff90c212815b60 24203 lru 00000080:00200000:1.0:1775764896.236258:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a89f47a3 24203: 1< 0 - - > 2< 99135488 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1478656 - | + - + - > 5< + - + - | 0 - | 3723 - -> 00000080:00200000:1.0:1775764896.236260:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c436d563 00000080:00200000:1.0:1775764896.236261:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.236263:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.236270:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.236278:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.236285:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.236292:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.236303:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.236305:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.236306:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.236314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.236316:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.236317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.236320:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236325:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.236332:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.236335:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.236339:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.236341:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24203, 24203] 00000080:00200000:1.0:1775764896.236352:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.236354:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.236382:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99139584 i_blocks 90112 00000080:00200000:1.0:1775764896.236386:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.236402:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99139584, count: 4096 00000080:00200000:1.0:1775764896.236412:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99139584, bytes: 4096 00000080:00200000:1.0:1775764896.236414:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.236416:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24205, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.236417:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99139584, 99143680) 0 0 00000080:00200000:1.0:1775764896.236419:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.236423:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24204, 24204] 00020000:00200000:1.0:1775764896.236425:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99139584 chunk: [99139584, 99143680) 99143680, 4096 00020000:00200000:1.0:1775764896.236427:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.236434:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24204, 24204] 00000008:00010000:1.0:1775764896.236437:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.236439:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.236447:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236455:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99139584 99143679) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236463:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236470:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236482:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.236490:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99139584, 99143680) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.236495:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24204 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.236510:0:18302:0:(rw26.c:858:ll_write_end()) pos 99139584, len 4096, copied 4096 00000080:00200000:1.0:1775764896.236515:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000038a6519d[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.236517:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004d16e18e 17ffffc0001001 3:0 ffff90c212815000 24204 lru 00000080:00200000:1.0:1775764896.236522:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a9fa1b81 24204: 1< 0 - - > 2< 99139584 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1474560 - | + - + - > 5< + - + - | 0 - | 3724 - -> 00000080:00200000:1.0:1775764896.236524:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000038a6519d 00000080:00200000:1.0:1775764896.236525:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.236527:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.236535:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.236543:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.236550:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.236558:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.236570:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.236573:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.236575:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.236584:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.236587:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.236589:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.236593:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236601:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.236608:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.236611:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.236614:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.236617:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24204, 24204] 00000080:00200000:1.0:1775764896.236628:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.236631:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.236659:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99143680 i_blocks 90112 00000080:00200000:1.0:1775764896.236662:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.236679:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99143680, count: 4096 00000080:00200000:1.0:1775764896.236690:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99143680, bytes: 4096 00000080:00200000:1.0:1775764896.236692:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.236696:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24206, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.236697:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99143680, 99147776) 0 0 00000080:00200000:1.0:1775764896.236699:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.236704:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24205, 24205] 00020000:00200000:1.0:1775764896.236707:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99143680 chunk: [99143680, 99147776) 99147776, 4096 00020000:00200000:1.0:1775764896.236709:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.236720:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24205, 24205] 00000008:00010000:1.0:1775764896.236725:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.236727:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.236735:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236743:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99143680 99147775) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236751:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236759:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236766:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.236774:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99143680, 99147776) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.236780:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24205 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.236795:0:18302:0:(rw26.c:858:ll_write_end()) pos 99143680, len 4096, copied 4096 00000080:00200000:1.0:1775764896.236799:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000264fae81[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.236802:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f7190b39 17ffffc0001001 3:0 ffff90c212815ee0 24205 lru 00000080:00200000:1.0:1775764896.236807:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d3f13ec2 24205: 1< 0 - - > 2< 99143680 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1470464 - | + - + - > 5< + - + - | 0 - | 3725 - -> 00000080:00200000:1.0:1775764896.236809:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000264fae81 00000080:00200000:1.0:1775764896.236810:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.236812:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.236820:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.236828:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.236835:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.236843:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.236855:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.236858:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.236860:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.236965:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.236969:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.236972:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.236976:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.236984:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.236992:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.236995:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.237000:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.237002:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24205, 24205] 00000080:00200000:1.0:1775764896.237013:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.237017:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.237048:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99147776 i_blocks 90112 00000080:00200000:1.0:1775764896.237052:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.237066:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99147776, count: 4096 00000080:00200000:1.0:1775764896.237073:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99147776, bytes: 4096 00000080:00200000:1.0:1775764896.237075:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.237077:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24207, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.237078:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99147776, 99151872) 0 0 00000080:00200000:1.0:1775764896.237080:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.237084:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24206, 24206] 00020000:00200000:1.0:1775764896.237086:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99147776 chunk: [99147776, 99151872) 99151872, 4096 00020000:00200000:1.0:1775764896.237088:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.237095:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24206, 24206] 00000008:00010000:1.0:1775764896.237098:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.237100:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.237106:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237111:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99147776 99151871) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237129:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237134:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237138:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.237143:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99147776, 99151872) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.237147:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24206 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.237160:0:18302:0:(rw26.c:858:ll_write_end()) pos 99147776, len 4096, copied 4096 00000080:00200000:1.0:1775764896.237163:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000491600ca[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.237165:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008e083d75 17ffffc0001001 3:0 ffff90c2128150e0 24206 lru 00000080:00200000:1.0:1775764896.237168:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000029d93a06 24206: 1< 0 - - > 2< 99147776 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1466368 - | + - + - > 5< + - + - | 0 - | 3726 - -> 00000080:00200000:1.0:1775764896.237170:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000491600ca 00000080:00200000:1.0:1775764896.237171:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.237172:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.237177:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237182:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237186:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237190:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.237197:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.237199:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.237200:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.237206:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.237208:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.237209:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.237212:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237216:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.237222:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.237224:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.237226:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.237228:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24206, 24206] 00000080:00200000:1.0:1775764896.237235:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.237237:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.237256:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99151872 i_blocks 90112 00000080:00200000:1.0:1775764896.237258:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.237269:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99151872, count: 4096 00000080:00200000:1.0:1775764896.237275:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99151872, bytes: 4096 00000080:00200000:1.0:1775764896.237277:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.237279:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24208, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.237280:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99151872, 99155968) 0 0 00000080:00200000:1.0:1775764896.237282:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.237285:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24207, 24207] 00020000:00200000:1.0:1775764896.237287:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99151872 chunk: [99151872, 99155968) 99155968, 4096 00020000:00200000:1.0:1775764896.237289:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.237295:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24207, 24207] 00000008:00010000:1.0:1775764896.237298:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.237300:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.237305:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237310:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99151872 99155967) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237316:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237324:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237330:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.237337:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99151872, 99155968) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.237341:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24207 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.237357:0:18302:0:(rw26.c:858:ll_write_end()) pos 99151872, len 4096, copied 4096 00000080:00200000:1.0:1775764896.237361:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000002826851[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.237364:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000031f1b0d0 17ffffc0001001 3:0 ffff90c2128151c0 24207 lru 00000080:00200000:1.0:1775764896.237369:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002bf8e6da 24207: 1< 0 - - > 2< 99151872 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1462272 - | + - + - > 5< + - + - | 0 - | 3727 - -> 00000080:00200000:1.0:1775764896.237371:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000002826851 00000080:00200000:1.0:1775764896.237372:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.237374:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.237382:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237390:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237398:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237405:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.237419:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.237421:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.237423:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.237430:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.237432:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.237434:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.237437:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237445:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.237453:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.237455:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.237459:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.237461:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24207, 24207] 00000080:00200000:1.0:1775764896.237471:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.237474:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.237502:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99155968 i_blocks 90112 00000080:00200000:1.0:1775764896.237506:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.237522:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99155968, count: 4096 00000080:00200000:1.0:1775764896.237532:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99155968, bytes: 4096 00000080:00200000:1.0:1775764896.237534:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.237538:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24209, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.237540:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99155968, 99160064) 0 0 00000080:00200000:1.0:1775764896.237542:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.237547:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24208, 24208] 00020000:00200000:1.0:1775764896.237551:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99155968 chunk: [99155968, 99160064) 99160064, 4096 00020000:00200000:1.0:1775764896.237553:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.237565:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24208, 24208] 00000008:00010000:1.0:1775764896.237570:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.237572:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.237580:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237589:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99155968 99160063) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237597:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237604:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237611:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.237618:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99155968, 99160064) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.237624:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24208 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.237651:0:18302:0:(rw26.c:858:ll_write_end()) pos 99155968, len 4096, copied 4096 00000080:00200000:1.0:1775764896.237656:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000033f6140d[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.237658:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000069a78f64 17ffffc0001001 3:0 ffff90c212828ee0 24208 lru 00000080:00200000:1.0:1775764896.237663:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002d531aea 24208: 1< 0 - - > 2< 99155968 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1458176 - | + - + - > 5< + - + - | 0 - | 3728 - -> 00000080:00200000:1.0:1775764896.237665:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000033f6140d 00000080:00200000:1.0:1775764896.237666:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.237669:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.237677:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237685:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237691:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237698:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.237708:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.237712:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.237714:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.237724:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.237726:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.237729:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.237733:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237740:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.237748:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.237750:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.237754:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.237757:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24208, 24208] 00000080:00200000:1.0:1775764896.237768:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.237771:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.237799:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99160064 i_blocks 90112 00000080:00200000:1.0:1775764896.237803:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.237817:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99160064, count: 4096 00000080:00200000:1.0:1775764896.237828:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99160064, bytes: 4096 00000080:00200000:1.0:1775764896.237831:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.237834:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24210, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.237836:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99160064, 99164160) 0 0 00000080:00200000:1.0:1775764896.237838:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.237844:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24209, 24209] 00020000:00200000:1.0:1775764896.237847:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99160064 chunk: [99160064, 99164160) 99164160, 4096 00020000:00200000:1.0:1775764896.237849:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.237860:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24209, 24209] 00000008:00010000:1.0:1775764896.237866:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.237868:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.237899:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237907:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99160064 99164159) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237916:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237923:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.237930:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.237938:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99160064, 99164160) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.237943:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24209 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.237959:0:18302:0:(rw26.c:858:ll_write_end()) pos 99160064, len 4096, copied 4096 00000080:00200000:1.0:1775764896.237963:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005a45abc4[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.237966:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ec4acfe4 17ffffc0001001 3:0 ffff90c2128280e0 24209 lru 00000080:00200000:1.0:1775764896.237971:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002b963fd7 24209: 1< 0 - - > 2< 99160064 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1454080 - | + - + - > 5< + - + - | 0 - | 3729 - -> 00000080:00200000:1.0:1775764896.237973:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005a45abc4 00000080:00200000:1.0:1775764896.237974:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.237976:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.237984:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237992:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.237999:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238006:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.238017:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.238021:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.238023:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.238032:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.238036:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.238038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.238042:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238048:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.238054:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.238055:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.238058:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.238060:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24209, 24209] 00000080:00200000:1.0:1775764896.238068:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.238069:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.238089:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99164160 i_blocks 90112 00000080:00200000:1.0:1775764896.238091:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.238102:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99164160, count: 4096 00000080:00200000:1.0:1775764896.238108:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99164160, bytes: 4096 00000080:00200000:1.0:1775764896.238110:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.238127:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24211, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.238129:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99164160, 99168256) 0 0 00000080:00200000:1.0:1775764896.238130:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.238135:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24210, 24210] 00020000:00200000:1.0:1775764896.238137:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99164160 chunk: [99164160, 99168256) 99168256, 4096 00020000:00200000:1.0:1775764896.238138:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.238146:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24210, 24210] 00000008:00010000:1.0:1775764896.238150:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.238151:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.238157:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238163:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99164160 99168255) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238169:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238174:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238177:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.238182:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99164160, 99168256) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.238186:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24210 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.238198:0:18302:0:(rw26.c:858:ll_write_end()) pos 99164160, len 4096, copied 4096 00000080:00200000:1.0:1775764896.238201:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e744d869[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.238202:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004a840f79 17ffffc0001001 3:0 ffff90c2128281c0 24210 lru 00000080:00200000:1.0:1775764896.238206:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000aa1a4d09 24210: 1< 0 - - > 2< 99164160 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1449984 - | + - + - > 5< + - + - | 0 - | 3730 - -> 00000080:00200000:1.0:1775764896.238208:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e744d869 00000080:00200000:1.0:1775764896.238208:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.238210:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.238215:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238220:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238224:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238232:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.238239:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.238241:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.238242:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.238248:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.238250:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.238251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.238254:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238259:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.238264:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.238265:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.238267:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.238269:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24210, 24210] 00000080:00200000:1.0:1775764896.238276:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.238278:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.238296:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99168256 i_blocks 90112 00000080:00200000:1.0:1775764896.238298:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.238309:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99168256, count: 4096 00000080:00200000:1.0:1775764896.238315:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99168256, bytes: 4096 00000080:00200000:1.0:1775764896.238317:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.238319:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24212, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.238321:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99168256, 99172352) 0 0 00000080:00200000:1.0:1775764896.238322:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.238326:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24211, 24211] 00020000:00200000:1.0:1775764896.238328:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99168256 chunk: [99168256, 99172352) 99172352, 4096 00020000:00200000:1.0:1775764896.238331:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.238341:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24211, 24211] 00000008:00010000:1.0:1775764896.238345:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.238347:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.238354:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238362:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99168256 99172351) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238370:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238377:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238384:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.238393:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99168256, 99172352) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.238398:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24211 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.238412:0:18302:0:(rw26.c:858:ll_write_end()) pos 99168256, len 4096, copied 4096 00000080:00200000:1.0:1775764896.238415:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006cd01f1f[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.238417:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000200d5d25 17ffffc0001001 3:0 ffff90c2128287e0 24211 lru 00000080:00200000:1.0:1775764896.238421:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002b083d61 24211: 1< 0 - - > 2< 99168256 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1445888 - | + - + - > 5< + - + - | 0 - | 3731 - -> 00000080:00200000:1.0:1775764896.238422:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006cd01f1f 00000080:00200000:1.0:1775764896.238423:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.238425:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.238430:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238435:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238439:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238443:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.238453:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.238456:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.238457:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.238466:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.238469:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.238471:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.238474:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238482:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.238490:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.238493:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.238497:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.238499:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24211, 24211] 00000080:00200000:1.0:1775764896.238510:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.238513:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.238542:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99172352 i_blocks 90112 00000080:00200000:1.0:1775764896.238545:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.238560:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99172352, count: 4096 00000080:00200000:1.0:1775764896.238570:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99172352, bytes: 4096 00000080:00200000:1.0:1775764896.238572:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.238576:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24213, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.238578:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99172352, 99176448) 0 0 00000080:00200000:1.0:1775764896.238580:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.238585:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24212, 24212] 00020000:00200000:1.0:1775764896.238589:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99172352 chunk: [99172352, 99176448) 99176448, 4096 00020000:00200000:1.0:1775764896.238591:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.238602:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24212, 24212] 00000008:00010000:1.0:1775764896.238606:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.238609:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.238616:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238625:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99172352 99176447) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238633:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238641:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238654:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.238662:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99172352, 99176448) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.238667:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24212 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.238685:0:18302:0:(rw26.c:858:ll_write_end()) pos 99172352, len 4096, copied 4096 00000080:00200000:1.0:1775764896.238689:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002a019aac[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.238691:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000007259ddc 17ffffc0001001 3:0 ffff90c2128289a0 24212 lru 00000080:00200000:1.0:1775764896.238697:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000076eb12b 24212: 1< 0 - - > 2< 99172352 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1441792 - | + - + - > 5< + - + - | 0 - | 3732 - -> 00000080:00200000:1.0:1775764896.238699:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002a019aac 00000080:00200000:1.0:1775764896.238700:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.238702:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.238710:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238718:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238725:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.238733:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.238744:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.238747:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.238749:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.238758:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.238761:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.238764:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.238768:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238776:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.238785:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.238787:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.238791:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.238794:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24212, 24212] 00000080:00200000:1.0:1775764896.238804:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.238807:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.238836:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99176448 i_blocks 90112 00000080:00200000:1.0:1775764896.238840:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.238855:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99176448, count: 4096 00000080:00200000:1.0:1775764896.238865:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99176448, bytes: 4096 00000080:00200000:1.0:1775764896.238868:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.238896:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24214, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.238898:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99176448, 99180544) 0 0 00000080:00200000:1.0:1775764896.238901:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.238906:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24213, 24213] 00020000:00200000:1.0:1775764896.238909:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99176448 chunk: [99176448, 99180544) 99180544, 4096 00020000:00200000:1.0:1775764896.238912:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.238924:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24213, 24213] 00000008:00010000:1.0:1775764896.238929:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.238931:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.238939:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238948:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99176448 99180543) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238956:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238963:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.238970:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.238978:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99176448, 99180544) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.238983:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24213 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.239005:0:18302:0:(rw26.c:858:ll_write_end()) pos 99176448, len 4096, copied 4096 00000080:00200000:1.0:1775764896.239010:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f8994d3f[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.239013:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001eb341fe 17ffffc0001021 2:0 ffff90c2128282a0 24213 lru 00000080:00200000:1.0:1775764896.239018:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000608190e7 24213: 1< 0 - - > 2< 99176448 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1437696 - | + - + - > 5< + - + - | 0 - | 3733 - -> 00000080:00200000:1.0:1775764896.239020:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f8994d3f 00000080:00200000:1.0:1775764896.239021:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.239023:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.239030:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239038:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239045:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239053:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.239062:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.239064:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.239066:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.239073:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.239075:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.239077:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.239080:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239085:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.239089:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.239091:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.239093:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.239095:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24213, 24213] 00000080:00200000:1.0:1775764896.239102:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.239104:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.239139:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99180544 i_blocks 90112 00000080:00200000:1.0:1775764896.239141:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.239153:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99180544, count: 4096 00000080:00200000:1.0:1775764896.239159:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99180544, bytes: 4096 00000080:00200000:1.0:1775764896.239160:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.239163:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24215, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.239164:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99180544, 99184640) 0 0 00000080:00200000:1.0:1775764896.239166:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.239169:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24214, 24214] 00020000:00200000:1.0:1775764896.239172:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99180544 chunk: [99180544, 99184640) 99184640, 4096 00020000:00200000:1.0:1775764896.239173:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.239180:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24214, 24214] 00000008:00010000:1.0:1775764896.239183:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.239185:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.239190:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239196:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99180544 99184639) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239202:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239207:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239210:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.239215:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99180544, 99184640) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.239218:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24214 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.239230:0:18302:0:(rw26.c:858:ll_write_end()) pos 99180544, len 4096, copied 4096 00000080:00200000:1.0:1775764896.239233:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000000fb54e3[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.239235:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000bc4db814 17ffffc0001001 3:0 ffff90c212828d20 24214 lru 00000080:00200000:1.0:1775764896.239239:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000074540020 24214: 1< 0 - - > 2< 99180544 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1433600 - | + - + - > 5< + - + - | 0 - | 3734 - -> 00000080:00200000:1.0:1775764896.239240:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000000fb54e3 00000080:00200000:1.0:1775764896.239241:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.239242:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.239247:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239252:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239256:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239260:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.239267:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.239268:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.239270:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.239276:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.239277:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.239278:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.239281:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239285:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.239291:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.239292:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.239295:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.239297:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24214, 24214] 00000080:00200000:1.0:1775764896.239304:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.239305:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.239324:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99184640 i_blocks 90112 00000080:00200000:1.0:1775764896.239327:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.239341:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99184640, count: 4096 00000080:00200000:1.0:1775764896.239350:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99184640, bytes: 4096 00000080:00200000:1.0:1775764896.239353:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.239356:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24216, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.239357:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99184640, 99188736) 0 0 00000080:00200000:1.0:1775764896.239360:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.239364:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24215, 24215] 00020000:00200000:1.0:1775764896.239367:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99184640 chunk: [99184640, 99188736) 99188736, 4096 00020000:00200000:1.0:1775764896.239369:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.239380:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24215, 24215] 00000008:00010000:1.0:1775764896.239385:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.239388:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.239396:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239405:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99184640 99188735) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239412:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239421:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239425:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.239430:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99184640, 99188736) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.239433:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24215 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.239446:0:18302:0:(rw26.c:858:ll_write_end()) pos 99184640, len 4096, copied 4096 00000080:00200000:1.0:1775764896.239450:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000048fe7991[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.239453:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000020986910 17ffffc0001001 3:0 ffff90c212828380 24215 lru 00000080:00200000:1.0:1775764896.239458:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000af1397e2 24215: 1< 0 - - > 2< 99184640 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1429504 - | + - + - > 5< + - + - | 0 - | 3735 - -> 00000080:00200000:1.0:1775764896.239459:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000048fe7991 00000080:00200000:1.0:1775764896.239460:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.239462:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.239470:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239477:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239483:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239491:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.239507:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.239510:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.239513:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.239522:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.239524:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.239527:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.239530:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239538:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.239546:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.239548:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.239552:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.239555:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24215, 24215] 00000080:00200000:1.0:1775764896.239566:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.239569:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.239598:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99188736 i_blocks 90112 00000080:00200000:1.0:1775764896.239602:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.239617:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99188736, count: 4096 00000080:00200000:1.0:1775764896.239627:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99188736, bytes: 4096 00000080:00200000:1.0:1775764896.239630:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.239633:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24217, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.239635:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99188736, 99192832) 0 0 00000080:00200000:1.0:1775764896.239637:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.239642:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24216, 24216] 00020000:00200000:1.0:1775764896.239646:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99188736 chunk: [99188736, 99192832) 99192832, 4096 00020000:00200000:1.0:1775764896.239648:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.239659:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24216, 24216] 00000008:00010000:1.0:1775764896.239664:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.239667:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.239674:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239683:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99188736 99192831) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239691:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239698:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239705:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.239713:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99188736, 99192832) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.239718:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24216 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.239734:0:18302:0:(rw26.c:858:ll_write_end()) pos 99188736, len 4096, copied 4096 00000080:00200000:1.0:1775764896.239738:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000789107d9[3 00000000b99c1f5d 2 1 00000000290babbe] 00000080:00200000:1.0:1775764896.239740:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000000b578b1 17ffffc0001001 3:0 ffff90c212828620 24216 lru 00000080:00200000:1.0:1775764896.239746:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005b7fb57c 24216: 1< 0 - - > 2< 99188736 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1425408 - | + - + - > 5< + - + - | 0 - | 3736 - -> 00000080:00200000:1.0:1775764896.239748:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000789107d9 00000080:00200000:1.0:1775764896.239749:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.239751:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.239759:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239767:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000808b7af8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239775:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.239783:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.239794:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.239796:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.239799:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.239808:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.239811:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.239813:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.239817:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.239824:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.239832:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ead58 (ffff90c2195eee18/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.239834:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eee18/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.239838:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.239841:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24216, 24216] 00000080:00200000:1.0:1775764896.239852:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.239855:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.239973:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99192832 i_blocks 90112 00000080:00200000:0.0:1775764896.239978:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.239999:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99192832, count: 4096 00000080:00200000:0.0:1775764896.240017:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99192832, bytes: 4096 00000080:00200000:0.0:1775764896.240020:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.240024:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24218, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.240026:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99192832, 99196928) 0 0 00000080:00200000:0.0:1775764896.240028:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.240033:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24217, 24217] 00020000:00200000:0.0:1775764896.240036:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99192832 chunk: [99192832, 99196928) 99196928, 4096 00020000:00200000:0.0:1775764896.240038:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.240054:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24217, 24217] 00000008:00010000:0.0:1775764896.240060:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.240062:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.240070:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240079:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99192832 99196927) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240087:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240093:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240099:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.240106:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99192832, 99196928) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.240111:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24217 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.240141:0:18302:0:(rw26.c:858:ll_write_end()) pos 99192832, len 4096, copied 4096 00000080:00200000:0.0:1775764896.240145:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c8ee86ee[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.240148:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002885559c 17ffffc0001001 3:0 ffff90c2128122a0 24217 lru 00000080:00200000:0.0:1775764896.240153:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a72f99e5 24217: 1< 0 - - > 2< 99192832 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1421312 - | + - + - > 5< + - + - | 0 - | 3737 - -> 00000080:00200000:0.0:1775764896.240156:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c8ee86ee 00000080:00200000:0.0:1775764896.240157:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.240159:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.240167:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240174:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240180:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240188:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.240199:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.240202:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.240204:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.240211:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.240214:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.240216:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.240219:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240227:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.240234:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.240237:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.240243:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.240245:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24217, 24217] 00000080:00200000:0.0:1775764896.240258:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.240260:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.240264:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99196928 i_blocks 90112 00000080:00200000:0.0:1775764896.240266:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.240279:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99196928, count: 4096 00000080:00200000:0.0:1775764896.240287:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99196928, bytes: 4096 00000080:00200000:0.0:1775764896.240288:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.240291:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24219, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.240292:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99196928, 99201024) 0 0 00000080:00200000:0.0:1775764896.240294:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.240297:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24218, 24218] 00020000:00200000:0.0:1775764896.240299:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99196928 chunk: [99196928, 99201024) 99201024, 4096 00020000:00200000:0.0:1775764896.240301:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.240307:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24218, 24218] 00000008:00010000:0.0:1775764896.240311:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.240312:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.240318:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240323:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99196928 99201023) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240331:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240338:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240349:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.240357:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99196928, 99201024) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.240361:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24218 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.240378:0:18302:0:(rw26.c:858:ll_write_end()) pos 99196928, len 4096, copied 4096 00000080:00200000:0.0:1775764896.240382:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ed7c712f[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.240385:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007e466a25 17ffffc0001001 3:0 ffff90c212812d20 24218 lru 00000080:00200000:0.0:1775764896.240391:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000aa056456 24218: 1< 0 - - > 2< 99196928 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1417216 - | + - + - > 5< + - + - | 0 - | 3738 - -> 00000080:00200000:0.0:1775764896.240393:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ed7c712f 00000080:00200000:0.0:1775764896.240394:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.240396:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.240404:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240411:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240416:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240420:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.240427:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.240429:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.240431:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.240435:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.240437:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.240440:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.240443:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240451:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.240458:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.240461:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.240465:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.240467:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24218, 24218] 00000080:00200000:0.0:1775764896.240477:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.240481:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.240485:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99201024 i_blocks 90112 00000080:00200000:0.0:1775764896.240488:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.240503:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99201024, count: 4096 00000080:00200000:0.0:1775764896.240515:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99201024, bytes: 4096 00000080:00200000:0.0:1775764896.240518:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.240521:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24220, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.240523:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99201024, 99205120) 0 0 00000080:00200000:0.0:1775764896.240525:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.240530:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24219, 24219] 00020000:00200000:0.0:1775764896.240533:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99201024 chunk: [99201024, 99205120) 99205120, 4096 00020000:00200000:0.0:1775764896.240535:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.240546:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24219, 24219] 00000008:00010000:0.0:1775764896.240551:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.240553:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.240561:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240570:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99201024 99205119) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240578:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240585:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240590:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.240598:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99201024, 99205120) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.240603:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24219 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.240618:0:18302:0:(rw26.c:858:ll_write_end()) pos 99201024, len 4096, copied 4096 00000080:00200000:0.0:1775764896.240623:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000018ab94c1[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.240626:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005542155f 17ffffc0001001 3:0 ffff90c212812380 24219 lru 00000080:00200000:0.0:1775764896.240631:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cb7fc040 24219: 1< 0 - - > 2< 99201024 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1413120 - | + - + - > 5< + - + - | 0 - | 3739 - -> 00000080:00200000:0.0:1775764896.240633:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000018ab94c1 00000080:00200000:0.0:1775764896.240634:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.240637:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.240644:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240652:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240659:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240666:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.240677:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.240680:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.240682:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.240688:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.240690:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.240692:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.240696:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240703:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.240711:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.240714:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.240718:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.240721:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24219, 24219] 00000080:00200000:0.0:1775764896.240730:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.240733:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.240738:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99205120 i_blocks 90112 00000080:00200000:0.0:1775764896.240740:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.240756:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99205120, count: 4096 00000080:00200000:0.0:1775764896.240767:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99205120, bytes: 4096 00000080:00200000:0.0:1775764896.240770:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.240773:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24221, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.240775:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99205120, 99209216) 0 0 00000080:00200000:0.0:1775764896.240777:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.240782:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24220, 24220] 00020000:00200000:0.0:1775764896.240785:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99205120 chunk: [99205120, 99209216) 99209216, 4096 00020000:00200000:0.0:1775764896.240787:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.240799:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24220, 24220] 00000008:00010000:0.0:1775764896.240804:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.240806:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.240813:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240823:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99205120 99209215) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240831:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240839:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240845:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.240853:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99205120, 99209216) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.240858:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24220 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.240896:0:18302:0:(rw26.c:858:ll_write_end()) pos 99205120, len 4096, copied 4096 00000080:00200000:0.0:1775764896.240901:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006267bda1[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.240904:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008fe83091 17ffffc0001001 3:0 ffff90c212812620 24220 lru 00000080:00200000:0.0:1775764896.240909:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d1641408 24220: 1< 0 - - > 2< 99205120 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1409024 - | + - + - > 5< + - + - | 0 - | 3740 - -> 00000080:00200000:0.0:1775764896.240911:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006267bda1 00000080:00200000:0.0:1775764896.240912:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.240914:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.240922:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240930:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240936:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.240943:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.240955:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.240958:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.240960:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.240966:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.240969:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.240971:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.240975:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.240983:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.240991:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.240993:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.240998:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.241000:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24220, 24220] 00000080:00200000:0.0:1775764896.241010:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.241014:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.241018:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99209216 i_blocks 90112 00000080:00200000:0.0:1775764896.241021:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.241033:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99209216, count: 4096 00000080:00200000:0.0:1775764896.241041:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99209216, bytes: 4096 00000080:00200000:0.0:1775764896.241042:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.241045:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24222, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.241046:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99209216, 99213312) 0 0 00000080:00200000:0.0:1775764896.241047:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.241051:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24221, 24221] 00020000:00200000:0.0:1775764896.241053:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99209216 chunk: [99209216, 99213312) 99213312, 4096 00020000:00200000:0.0:1775764896.241054:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.241061:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24221, 24221] 00000008:00010000:0.0:1775764896.241064:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.241066:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.241071:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241077:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99209216 99213311) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241083:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241091:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241096:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.241100:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99209216, 99213312) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.241103:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24221 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.241129:0:18302:0:(rw26.c:858:ll_write_end()) pos 99209216, len 4096, copied 4096 00000080:00200000:0.0:1775764896.241132:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000089ba4daa[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.241134:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000068528dbb 17ffffc0001001 3:0 ffff90c212812e00 24221 lru 00000080:00200000:0.0:1775764896.241137:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009ba5d06b 24221: 1< 0 - - > 2< 99209216 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1404928 - | + - + - > 5< + - + - | 0 - | 3741 - -> 00000080:00200000:0.0:1775764896.241138:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000089ba4daa 00000080:00200000:0.0:1775764896.241139:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.241140:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.241145:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241150:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241154:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241158:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.241169:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.241171:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.241173:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.241177:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.241178:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.241180:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.241183:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241188:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.241193:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.241195:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.241197:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.241199:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24221, 24221] 00000080:00200000:0.0:1775764896.241206:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.241208:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.241211:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99213312 i_blocks 90112 00000080:00200000:0.0:1775764896.241213:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.241223:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99213312, count: 4096 00000080:00200000:0.0:1775764896.241230:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99213312, bytes: 4096 00000080:00200000:0.0:1775764896.241232:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.241234:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24223, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.241235:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99213312, 99217408) 0 0 00000080:00200000:0.0:1775764896.241237:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.241240:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24222, 24222] 00020000:00200000:0.0:1775764896.241242:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99213312 chunk: [99213312, 99217408) 99217408, 4096 00020000:00200000:0.0:1775764896.241243:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.241249:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24222, 24222] 00000008:00010000:0.0:1775764896.241252:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.241254:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.241259:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241264:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99213312 99217407) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241269:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241274:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241278:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.241282:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99213312, 99217408) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.241285:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24222 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.241302:0:18302:0:(rw26.c:858:ll_write_end()) pos 99213312, len 4096, copied 4096 00000080:00200000:0.0:1775764896.241305:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f2777d3b[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.241306:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ea2ceb53 17ffffc0001021 2:0 ffff90c212812540 24222 lru 00000080:00200000:0.0:1775764896.241310:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ad4d69f3 24222: 1< 0 - - > 2< 99213312 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1400832 - | + - + - > 5< + - + - | 0 - | 3742 - -> 00000080:00200000:0.0:1775764896.241311:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f2777d3b 00000080:00200000:0.0:1775764896.241312:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.241314:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.241319:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241324:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241330:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241337:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.241350:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.241353:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.241356:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.241361:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.241363:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.241365:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.241369:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241377:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.241385:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.241388:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.241392:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.241394:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24222, 24222] 00000080:00200000:0.0:1775764896.241405:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.241408:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.241411:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99217408 i_blocks 90112 00000080:00200000:0.0:1775764896.241413:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.241424:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99217408, count: 4096 00000080:00200000:0.0:1775764896.241431:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99217408, bytes: 4096 00000080:00200000:0.0:1775764896.241433:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.241435:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24224, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.241436:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99217408, 99221504) 0 0 00000080:00200000:0.0:1775764896.241438:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.241443:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24223, 24223] 00020000:00200000:0.0:1775764896.241446:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99217408 chunk: [99217408, 99221504) 99221504, 4096 00020000:00200000:0.0:1775764896.241448:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.241459:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24223, 24223] 00000008:00010000:0.0:1775764896.241464:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.241466:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.241473:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241482:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99217408 99221503) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241491:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241498:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241506:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.241515:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99217408, 99221504) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.241520:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24223 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.241536:0:18302:0:(rw26.c:858:ll_write_end()) pos 99217408, len 4096, copied 4096 00000080:00200000:0.0:1775764896.241540:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c528d4bb[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.241543:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b1247e70 17ffffc0001001 3:0 ffff90c212812460 24223 lru 00000080:00200000:0.0:1775764896.241548:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d77529d5 24223: 1< 0 - - > 2< 99217408 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1396736 - | + - + - > 5< + - + - | 0 - | 3743 - -> 00000080:00200000:0.0:1775764896.241550:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c528d4bb 00000080:00200000:0.0:1775764896.241552:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.241554:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.241561:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241569:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241576:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241584:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.241594:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.241597:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.241599:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.241605:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.241607:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.241609:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.241614:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241622:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.241630:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.241632:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.241636:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.241639:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24223, 24223] 00000080:00200000:0.0:1775764896.241649:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.241652:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.241657:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99221504 i_blocks 90112 00000080:00200000:0.0:1775764896.241659:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.241674:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99221504, count: 4096 00000080:00200000:0.0:1775764896.241685:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99221504, bytes: 4096 00000080:00200000:0.0:1775764896.241688:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.241691:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24225, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.241692:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99221504, 99225600) 0 0 00000080:00200000:0.0:1775764896.241695:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.241699:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24224, 24224] 00020000:00200000:0.0:1775764896.241702:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99221504 chunk: [99221504, 99225600) 99225600, 4096 00020000:00200000:0.0:1775764896.241704:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.241716:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24224, 24224] 00000008:00010000:0.0:1775764896.241721:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.241723:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.241730:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241739:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99221504 99225599) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241747:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241755:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241762:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.241770:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99221504, 99225600) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.241776:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24224 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.241803:0:18302:0:(rw26.c:858:ll_write_end()) pos 99221504, len 4096, copied 4096 00000080:00200000:0.0:1775764896.241807:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ff62b788[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.241810:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000072198df8 17ffffc0001001 3:0 ffff90c21282b000 24224 lru 00000080:00200000:0.0:1775764896.241816:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000005f46600 24224: 1< 0 - - > 2< 99221504 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1392640 - | + - + - > 5< + - + - | 0 - | 3744 - -> 00000080:00200000:0.0:1775764896.241818:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ff62b788 00000080:00200000:0.0:1775764896.241819:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.241821:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.241829:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241837:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241843:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.241855:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.241866:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.241892:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.241894:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.241901:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.241903:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.241906:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.241910:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.241917:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.241924:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.241927:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.241931:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.241933:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24224, 24224] 00000080:00200000:0.0:1775764896.241944:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.241947:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.241951:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99225600 i_blocks 90112 00000080:00200000:0.0:1775764896.241954:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.241969:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99225600, count: 4096 00000080:00200000:0.0:1775764896.241980:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99225600, bytes: 4096 00000080:00200000:0.0:1775764896.241983:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.241987:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24226, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.241989:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99225600, 99229696) 0 0 00000080:00200000:0.0:1775764896.241991:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.241997:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24225, 24225] 00020000:00200000:0.0:1775764896.242000:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99225600 chunk: [99225600, 99229696) 99229696, 4096 00020000:00200000:0.0:1775764896.242002:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.242013:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24225, 24225] 00000008:00010000:0.0:1775764896.242019:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.242021:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.242029:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242037:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99225600 99229695) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242043:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242048:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242051:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.242057:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99225600, 99229696) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.242061:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24225 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.242073:0:18302:0:(rw26.c:858:ll_write_end()) pos 99225600, len 4096, copied 4096 00000080:00200000:0.0:1775764896.242075:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000620716b9[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.242077:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000ad54616 17ffffc0001001 3:0 ffff90c21282bee0 24225 lru 00000080:00200000:0.0:1775764896.242081:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003034d791 24225: 1< 0 - - > 2< 99225600 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1388544 - | + - + - > 5< + - + - | 0 - | 3745 - -> 00000080:00200000:0.0:1775764896.242083:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000620716b9 00000080:00200000:0.0:1775764896.242083:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.242085:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.242090:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242095:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242099:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242103:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.242110:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.242112:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.242127:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.242134:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.242136:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.242141:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.242145:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242151:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.242159:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.242161:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.242166:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.242169:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24225, 24225] 00000080:00200000:0.0:1775764896.242178:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.242181:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.242187:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99229696 i_blocks 90112 00000080:00200000:0.0:1775764896.242189:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.242204:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99229696, count: 4096 00000080:00200000:0.0:1775764896.242214:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99229696, bytes: 4096 00000080:00200000:0.0:1775764896.242217:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.242221:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24227, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.242222:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99229696, 99233792) 0 0 00000080:00200000:0.0:1775764896.242224:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.242229:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24226, 24226] 00020000:00200000:0.0:1775764896.242231:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99229696 chunk: [99229696, 99233792) 99233792, 4096 00020000:00200000:0.0:1775764896.242233:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.242243:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24226, 24226] 00000008:00010000:0.0:1775764896.242247:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.242249:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.242256:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242265:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99229696 99233791) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242273:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242278:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242286:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.242291:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99229696, 99233792) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.242295:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24226 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.242307:0:18302:0:(rw26.c:858:ll_write_end()) pos 99229696, len 4096, copied 4096 00000080:00200000:0.0:1775764896.242309:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007047f0c9[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.242311:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000794606f0 17ffffc0001001 3:0 ffff90c21282b0e0 24226 lru 00000080:00200000:0.0:1775764896.242314:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009fba623e 24226: 1< 0 - - > 2< 99229696 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1384448 - | + - + - > 5< + - + - | 0 - | 3746 - -> 00000080:00200000:0.0:1775764896.242316:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007047f0c9 00000080:00200000:0.0:1775764896.242317:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.242318:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.242323:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242329:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242337:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242344:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.242354:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.242357:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.242360:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.242374:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.242377:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.242379:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.242383:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242392:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.242400:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.242403:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.242407:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.242409:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24226, 24226] 00000080:00200000:0.0:1775764896.242416:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.242418:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.242446:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99233792 i_blocks 90112 00000080:00200000:0.0:1775764896.242450:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.242466:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99233792, count: 4096 00000080:00200000:0.0:1775764896.242476:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99233792, bytes: 4096 00000080:00200000:0.0:1775764896.242479:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.242482:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24228, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.242484:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99233792, 99237888) 0 0 00000080:00200000:0.0:1775764896.242486:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.242491:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24227, 24227] 00020000:00200000:0.0:1775764896.242494:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99233792 chunk: [99233792, 99237888) 99237888, 4096 00020000:00200000:0.0:1775764896.242496:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.242508:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24227, 24227] 00000008:00010000:0.0:1775764896.242513:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.242516:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.242524:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242531:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99233792 99237887) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242539:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242547:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242553:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.242561:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99233792, 99237888) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.242566:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24227 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.242583:0:18302:0:(rw26.c:858:ll_write_end()) pos 99233792, len 4096, copied 4096 00000080:00200000:0.0:1775764896.242587:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002db7d7ee[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.242590:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f58b2673 17ffffc0001001 3:0 ffff90c21282b1c0 24227 lru 00000080:00200000:0.0:1775764896.242595:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000086b5177e 24227: 1< 0 - - > 2< 99233792 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1380352 - | + - + - > 5< + - + - | 0 - | 3747 - -> 00000080:00200000:0.0:1775764896.242597:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002db7d7ee 00000080:00200000:0.0:1775764896.242598:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.242600:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.242607:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242615:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242622:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242629:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.242641:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.242644:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.242646:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.242656:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.242658:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.242660:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.242664:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242672:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.242680:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.242683:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.242687:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.242690:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24227, 24227] 00000080:00200000:0.0:1775764896.242700:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.242704:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.242736:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99237888 i_blocks 90112 00000080:00200000:0.0:1775764896.242739:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.242755:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99237888, count: 4096 00000080:00200000:0.0:1775764896.242766:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99237888, bytes: 4096 00000080:00200000:0.0:1775764896.242769:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.242773:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24229, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.242775:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99237888, 99241984) 0 0 00000080:00200000:0.0:1775764896.242777:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.242781:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24228, 24228] 00020000:00200000:0.0:1775764896.242784:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99237888 chunk: [99237888, 99241984) 99241984, 4096 00020000:00200000:0.0:1775764896.242786:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.242797:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24228, 24228] 00000008:00010000:0.0:1775764896.242802:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.242805:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.242812:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242821:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99237888 99241983) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242830:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242838:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.242845:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.242852:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99237888, 99241984) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.242857:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24228 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.242951:0:18302:0:(rw26.c:858:ll_write_end()) pos 99237888, len 4096, copied 4096 00000080:00200000:0.0:1775764896.242956:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009a21b0b7[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.242959:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008235920d 17ffffc0001001 3:0 ffff90c21282b7e0 24228 lru 00000080:00200000:0.0:1775764896.242965:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000eb080288 24228: 1< 0 - - > 2< 99237888 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1376256 - | + - + - > 5< + - + - | 0 - | 3748 - -> 00000080:00200000:0.0:1775764896.242968:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009a21b0b7 00000080:00200000:0.0:1775764896.242969:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.242971:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.242979:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242987:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.242993:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243000:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.243011:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.243014:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.243016:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.243025:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.243028:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.243030:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.243034:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243042:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.243048:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.243049:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.243053:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.243055:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24228, 24228] 00000080:00200000:0.0:1775764896.243062:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.243064:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.243084:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99241984 i_blocks 90112 00000080:00200000:0.0:1775764896.243087:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.243097:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99241984, count: 4096 00000080:00200000:0.0:1775764896.243104:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99241984, bytes: 4096 00000080:00200000:0.0:1775764896.243105:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.243108:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24230, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.243109:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99241984, 99246080) 0 0 00000080:00200000:0.0:1775764896.243111:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.243126:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24229, 24229] 00020000:00200000:0.0:1775764896.243128:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99241984 chunk: [99241984, 99246080) 99246080, 4096 00020000:00200000:0.0:1775764896.243130:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.243138:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24229, 24229] 00000008:00010000:0.0:1775764896.243141:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.243143:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.243148:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243154:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99241984 99246079) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243160:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243168:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243172:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.243177:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99241984, 99246080) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.243180:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24229 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.243192:0:18302:0:(rw26.c:858:ll_write_end()) pos 99241984, len 4096, copied 4096 00000080:00200000:0.0:1775764896.243194:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000019acadf2[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.243196:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c772e4b0 17ffffc0001001 3:0 ffff90c21282b9a0 24229 lru 00000080:00200000:0.0:1775764896.243199:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000085434e91 24229: 1< 0 - - > 2< 99241984 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1372160 - | + - + - > 5< + - + - | 0 - | 3749 - -> 00000080:00200000:0.0:1775764896.243200:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000019acadf2 00000080:00200000:0.0:1775764896.243201:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.243202:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.243207:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243212:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243216:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243220:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.243231:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.243232:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.243234:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.243240:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.243242:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.243244:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.243246:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243251:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.243256:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.243257:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.243260:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.243262:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24229, 24229] 00000080:00200000:0.0:1775764896.243268:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.243270:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.243288:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99246080 i_blocks 90112 00000080:00200000:0.0:1775764896.243291:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.243301:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99246080, count: 4096 00000080:00200000:0.0:1775764896.243307:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99246080, bytes: 4096 00000080:00200000:0.0:1775764896.243309:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.243311:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24231, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.243312:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99246080, 99250176) 0 0 00000080:00200000:0.0:1775764896.243314:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.243317:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24230, 24230] 00020000:00200000:0.0:1775764896.243319:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99246080 chunk: [99246080, 99250176) 99250176, 4096 00020000:00200000:0.0:1775764896.243320:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.243327:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24230, 24230] 00000008:00010000:0.0:1775764896.243332:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.243334:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.243342:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243350:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99246080 99250175) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243358:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243365:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243372:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.243380:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99246080, 99250176) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.243385:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24230 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.243401:0:18302:0:(rw26.c:858:ll_write_end()) pos 99246080, len 4096, copied 4096 00000080:00200000:0.0:1775764896.243405:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c3b5b1b2[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.243408:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003ad0808f 17ffffc0001001 3:0 ffff90c21282b2a0 24230 lru 00000080:00200000:0.0:1775764896.243412:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000083459f45 24230: 1< 0 - - > 2< 99246080 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1368064 - | + - + - > 5< + - + - | 0 - | 3750 - -> 00000080:00200000:0.0:1775764896.243414:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c3b5b1b2 00000080:00200000:0.0:1775764896.243414:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.243416:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.243422:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243427:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243431:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243435:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.243444:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.243447:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.243449:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.243457:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.243460:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.243462:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.243465:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243473:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.243481:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.243484:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.243488:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.243491:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24230, 24230] 00000080:00200000:0.0:1775764896.243501:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.243505:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.243533:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99250176 i_blocks 90112 00000080:00200000:0.0:1775764896.243537:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.243552:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99250176, count: 4096 00000080:00200000:0.0:1775764896.243563:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99250176, bytes: 4096 00000080:00200000:0.0:1775764896.243566:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.243569:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24232, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.243571:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99250176, 99254272) 0 0 00000080:00200000:0.0:1775764896.243573:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.243578:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24231, 24231] 00020000:00200000:0.0:1775764896.243581:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99250176 chunk: [99250176, 99254272) 99254272, 4096 00020000:00200000:0.0:1775764896.243583:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.243594:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24231, 24231] 00000008:00010000:0.0:1775764896.243599:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.243601:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.243608:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243616:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99250176 99254271) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243625:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243632:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243640:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.243648:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99250176, 99254272) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.243653:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24231 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.243669:0:18302:0:(rw26.c:858:ll_write_end()) pos 99250176, len 4096, copied 4096 00000080:00200000:0.0:1775764896.243673:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b915548e[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.243676:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d885e4ab 17ffffc0001001 3:0 ffff90c21282bd20 24231 lru 00000080:00200000:0.0:1775764896.243681:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b1a692b8 24231: 1< 0 - - > 2< 99250176 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1363968 - | + - + - > 5< + - + - | 0 - | 3751 - -> 00000080:00200000:0.0:1775764896.243683:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b915548e 00000080:00200000:0.0:1775764896.243684:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.243686:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.243693:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243701:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243708:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.243715:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.243726:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.243729:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.243731:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.243740:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.243742:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.243745:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.243749:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243757:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.243766:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.243768:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.243772:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.243774:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24231, 24231] 00000080:00200000:0.0:1775764896.243784:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.243787:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.243815:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99254272 i_blocks 90112 00000080:00200000:0.0:1775764896.243819:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.243835:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99254272, count: 4096 00000080:00200000:0.0:1775764896.243845:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99254272, bytes: 4096 00000080:00200000:0.0:1775764896.243848:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.243851:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24233, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.243853:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99254272, 99258368) 0 0 00000080:00200000:0.0:1775764896.243855:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.243860:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24232, 24232] 00020000:00200000:0.0:1775764896.243863:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99254272 chunk: [99254272, 99258368) 99258368, 4096 00020000:00200000:0.0:1775764896.243865:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.243899:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24232, 24232] 00000008:00010000:0.0:1775764896.243904:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.243906:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.243914:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243923:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99254272 99258367) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243931:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243939:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.243946:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.243954:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99254272, 99258368) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.243959:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24232 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.243974:0:18302:0:(rw26.c:858:ll_write_end()) pos 99254272, len 4096, copied 4096 00000080:00200000:0.0:1775764896.243979:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c51d5647[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.243981:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001f063acd 17ffffc0001001 3:0 ffff90c21282b380 24232 lru 00000080:00200000:0.0:1775764896.243986:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ecdd7a38 24232: 1< 0 - - > 2< 99254272 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1359872 - | + - + - > 5< + - + - | 0 - | 3752 - -> 00000080:00200000:0.0:1775764896.243988:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c51d5647 00000080:00200000:0.0:1775764896.243989:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.243991:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.243999:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244007:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244014:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244026:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.244037:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.244040:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.244042:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.244051:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.244054:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.244056:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.244060:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244067:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.244075:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.244077:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.244081:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.244083:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24232, 24232] 00000080:00200000:0.0:1775764896.244092:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.244095:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.244137:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99258368 i_blocks 90112 00000080:00200000:0.0:1775764896.244140:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.244151:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99258368, count: 4096 00000080:00200000:0.0:1775764896.244157:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99258368, bytes: 4096 00000080:00200000:0.0:1775764896.244159:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.244162:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24234, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.244163:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99258368, 99262464) 0 0 00000080:00200000:0.0:1775764896.244165:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.244169:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24233, 24233] 00020000:00200000:0.0:1775764896.244171:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99258368 chunk: [99258368, 99262464) 99262464, 4096 00020000:00200000:0.0:1775764896.244172:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.244179:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24233, 24233] 00000008:00010000:0.0:1775764896.244182:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.244184:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.244190:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244195:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99258368 99262463) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244201:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244205:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244209:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.244213:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99258368, 99262464) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.244216:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24233 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.244229:0:18302:0:(rw26.c:858:ll_write_end()) pos 99258368, len 4096, copied 4096 00000080:00200000:0.0:1775764896.244232:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004229801e[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.244233:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e6cab4d7 17ffffc0001001 3:0 ffff90c21282b620 24233 lru 00000080:00200000:0.0:1775764896.244237:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000abee7d79 24233: 1< 0 - - > 2< 99258368 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1355776 - | + - + - > 5< + - + - | 0 - | 3753 - -> 00000080:00200000:0.0:1775764896.244238:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004229801e 00000080:00200000:0.0:1775764896.244239:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.244241:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.244246:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244251:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244255:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244259:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.244266:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.244267:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.244269:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.244275:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.244277:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.244278:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.244280:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244285:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.244290:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.244292:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.244294:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.244296:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24233, 24233] 00000080:00200000:0.0:1775764896.244303:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.244305:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.244323:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99262464 i_blocks 90112 00000080:00200000:0.0:1775764896.244326:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.244341:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99262464, count: 4096 00000080:00200000:0.0:1775764896.244350:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99262464, bytes: 4096 00000080:00200000:0.0:1775764896.244353:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.244357:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24235, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.244358:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99262464, 99266560) 0 0 00000080:00200000:0.0:1775764896.244360:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.244365:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24234, 24234] 00020000:00200000:0.0:1775764896.244367:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99262464 chunk: [99262464, 99266560) 99266560, 4096 00020000:00200000:0.0:1775764896.244370:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.244380:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24234, 24234] 00000008:00010000:0.0:1775764896.244385:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.244388:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.244395:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244404:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99262464 99266559) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244411:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244416:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244424:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.244430:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99262464, 99266560) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.244433:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24234 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.244447:0:18302:0:(rw26.c:858:ll_write_end()) pos 99262464, len 4096, copied 4096 00000080:00200000:0.0:1775764896.244452:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000022e300c0[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.244454:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000048db0c95 17ffffc0001001 3:0 ffff90c21282be00 24234 lru 00000080:00200000:0.0:1775764896.244459:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b206ecb6 24234: 1< 0 - - > 2< 99262464 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1351680 - | + - + - > 5< + - + - | 0 - | 3754 - -> 00000080:00200000:0.0:1775764896.244461:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000022e300c0 00000080:00200000:0.0:1775764896.244462:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.244464:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.244471:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244479:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244486:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244494:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.244505:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.244508:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.244511:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.244520:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.244522:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.244524:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.244528:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244536:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.244544:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.244546:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.244550:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.244553:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24234, 24234] 00000080:00200000:0.0:1775764896.244564:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.244567:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.244595:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99266560 i_blocks 90112 00000080:00200000:0.0:1775764896.244598:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.244614:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99266560, count: 4096 00000080:00200000:0.0:1775764896.244624:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99266560, bytes: 4096 00000080:00200000:0.0:1775764896.244627:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.244630:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24236, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.244632:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99266560, 99270656) 0 0 00000080:00200000:0.0:1775764896.244634:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.244639:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24235, 24235] 00020000:00200000:0.0:1775764896.244642:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99266560 chunk: [99266560, 99270656) 99270656, 4096 00020000:00200000:0.0:1775764896.244644:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.244655:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24235, 24235] 00000008:00010000:0.0:1775764896.244661:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.244663:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.244670:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244679:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99266560 99270655) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244687:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244695:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244702:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.244710:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99266560, 99270656) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.244715:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24235 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.244731:0:18302:0:(rw26.c:858:ll_write_end()) pos 99266560, len 4096, copied 4096 00000080:00200000:0.0:1775764896.244735:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000cbb54f7[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.244737:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000364ca2d7 17ffffc0001001 3:0 ffff90c21282b540 24235 lru 00000080:00200000:0.0:1775764896.244743:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000046399d82 24235: 1< 0 - - > 2< 99266560 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1347584 - | + - + - > 5< + - + - | 0 - | 3755 - -> 00000080:00200000:0.0:1775764896.244745:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000cbb54f7 00000080:00200000:0.0:1775764896.244746:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.244748:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.244755:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244764:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244771:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.244778:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.244789:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.244791:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.244794:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.244803:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.244805:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.244807:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.244811:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.244818:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.244827:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.244829:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.244833:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.244836:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24235, 24235] 00000080:00200000:0.0:1775764896.244846:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.244849:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.244917:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99270656 i_blocks 90112 00000080:00200000:0.0:1775764896.244921:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.244938:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99270656, count: 4096 00000080:00200000:0.0:1775764896.244949:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99270656, bytes: 4096 00000080:00200000:0.0:1775764896.244952:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.244955:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24237, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.244957:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99270656, 99274752) 0 0 00000080:00200000:0.0:1775764896.244959:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.244965:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24236, 24236] 00020000:00200000:0.0:1775764896.244967:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99270656 chunk: [99270656, 99274752) 99274752, 4096 00020000:00200000:0.0:1775764896.244970:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.244981:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24236, 24236] 00000008:00010000:0.0:1775764896.244986:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.244988:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.244995:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245004:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99270656 99274751) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245012:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245019:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245025:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.245033:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99270656, 99274752) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.245038:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24236 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.245056:0:18302:0:(rw26.c:858:ll_write_end()) pos 99270656, len 4096, copied 4096 00000080:00200000:0.0:1775764896.245059:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000005d6e68f[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.245062:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c0971d5b 17ffffc0001001 3:0 ffff90c21282b460 24236 lru 00000080:00200000:0.0:1775764896.245067:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e178fbb2 24236: 1< 0 - - > 2< 99270656 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1343488 - | + - + - > 5< + - + - | 0 - | 3756 - -> 00000080:00200000:0.0:1775764896.245069:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000005d6e68f 00000080:00200000:0.0:1775764896.245069:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.245071:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.245079:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245088:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245094:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245101:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.245122:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.245126:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.245128:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.245136:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.245138:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.245139:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.245142:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245147:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.245153:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.245154:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.245157:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.245159:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24236, 24236] 00000080:00200000:0.0:1775764896.245166:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.245168:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.245187:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99274752 i_blocks 90112 00000080:00200000:0.0:1775764896.245190:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.245200:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99274752, count: 4096 00000080:00200000:0.0:1775764896.245206:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99274752, bytes: 4096 00000080:00200000:0.0:1775764896.245208:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.245210:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24238, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.245212:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99274752, 99278848) 0 0 00000080:00200000:0.0:1775764896.245213:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.245217:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24237, 24237] 00020000:00200000:0.0:1775764896.245219:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99274752 chunk: [99274752, 99278848) 99278848, 4096 00020000:00200000:0.0:1775764896.245220:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.245227:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24237, 24237] 00000008:00010000:0.0:1775764896.245230:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.245232:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.245237:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245243:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99274752 99278847) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245248:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245256:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245260:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.245264:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99274752, 99278848) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.245268:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24237 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.245285:0:18302:0:(rw26.c:858:ll_write_end()) pos 99274752, len 4096, copied 4096 00000080:00200000:0.0:1775764896.245287:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b14eb54b[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.245289:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d8044ec3 17ffffc0001021 2:0 ffff90c21282ba80 24237 lru 00000080:00200000:0.0:1775764896.245293:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004aca9d05 24237: 1< 0 - - > 2< 99274752 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1339392 - | + - + - > 5< + - + - | 0 - | 3757 - -> 00000080:00200000:0.0:1775764896.245295:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b14eb54b 00000080:00200000:0.0:1775764896.245296:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.245298:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.245305:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245312:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245318:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245324:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.245341:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.245343:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.245345:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.245355:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.245358:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.245360:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.245363:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245370:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.245378:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.245380:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.245384:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.245386:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24237, 24237] 00000080:00200000:0.0:1775764896.245395:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.245398:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.245427:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99278848 i_blocks 90112 00000080:00200000:0.0:1775764896.245431:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.245446:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99278848, count: 4096 00000080:00200000:0.0:1775764896.245456:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99278848, bytes: 4096 00000080:00200000:0.0:1775764896.245458:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.245462:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24239, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.245463:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99278848, 99282944) 0 0 00000080:00200000:0.0:1775764896.245465:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.245470:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24238, 24238] 00020000:00200000:0.0:1775764896.245473:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99278848 chunk: [99278848, 99282944) 99282944, 4096 00020000:00200000:0.0:1775764896.245475:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.245485:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24238, 24238] 00000008:00010000:0.0:1775764896.245490:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.245493:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.245500:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245508:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99278848 99282943) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245515:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245522:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245529:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.245537:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99278848, 99282944) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.245542:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24238 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.245558:0:18302:0:(rw26.c:858:ll_write_end()) pos 99278848, len 4096, copied 4096 00000080:00200000:0.0:1775764896.245562:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007c1672b6[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.245564:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b463d0d0 17ffffc0001001 3:0 ffff90c21282b700 24238 lru 00000080:00200000:0.0:1775764896.245569:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a5e58447 24238: 1< 0 - - > 2< 99278848 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1335296 - | + - + - > 5< + - + - | 0 - | 3758 - -> 00000080:00200000:0.0:1775764896.245572:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007c1672b6 00000080:00200000:0.0:1775764896.245573:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.245575:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.245582:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245591:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245598:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245606:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.245617:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.245620:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.245623:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.245633:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.245635:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.245637:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.245641:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245648:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.245656:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.245658:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.245663:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.245665:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24238, 24238] 00000080:00200000:0.0:1775764896.245676:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.245679:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.245708:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99282944 i_blocks 90112 00000080:00200000:0.0:1775764896.245711:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.245727:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99282944, count: 4096 00000080:00200000:0.0:1775764896.245737:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99282944, bytes: 4096 00000080:00200000:0.0:1775764896.245740:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.245743:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24240, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.245745:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99282944, 99287040) 0 0 00000080:00200000:0.0:1775764896.245747:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.245752:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24239, 24239] 00020000:00200000:0.0:1775764896.245755:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99282944 chunk: [99282944, 99287040) 99287040, 4096 00020000:00200000:0.0:1775764896.245757:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.245768:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24239, 24239] 00000008:00010000:0.0:1775764896.245773:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.245776:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.245783:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245792:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99282944 99287039) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245801:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245809:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245815:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.245823:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99282944, 99287040) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.245827:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24239 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.245843:0:18302:0:(rw26.c:858:ll_write_end()) pos 99282944, len 4096, copied 4096 00000080:00200000:0.0:1775764896.245847:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007858851d[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.245850:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003f14c1ea 17ffffc0001001 3:0 ffff90c21282bc40 24239 lru 00000080:00200000:0.0:1775764896.245855:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cdf717b4 24239: 1< 0 - - > 2< 99282944 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1331200 - | + - + - > 5< + - + - | 0 - | 3759 - -> 00000080:00200000:0.0:1775764896.245858:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007858851d 00000080:00200000:0.0:1775764896.245859:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.245861:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.245890:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245898:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245905:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.245913:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.245925:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.245928:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.245930:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.245939:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.245942:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.245944:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.245949:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.245956:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.245963:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.245966:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.245970:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.245973:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24239, 24239] 00000080:00200000:0.0:1775764896.245983:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.245986:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.246016:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99287040 i_blocks 90112 00000080:00200000:0.0:1775764896.246019:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.246034:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99287040, count: 4096 00000080:00200000:0.0:1775764896.246045:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99287040, bytes: 4096 00000080:00200000:0.0:1775764896.246047:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.246051:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24241, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.246053:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99287040, 99291136) 0 0 00000080:00200000:0.0:1775764896.246055:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.246060:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24240, 24240] 00020000:00200000:0.0:1775764896.246063:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99287040 chunk: [99287040, 99291136) 99291136, 4096 00020000:00200000:0.0:1775764896.246065:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.246077:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24240, 24240] 00000008:00010000:0.0:1775764896.246082:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.246084:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.246092:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246101:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99287040 99291135) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246109:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246200:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246206:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.246212:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99287040, 99291136) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.246216:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24240 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.246228:0:18302:0:(rw26.c:858:ll_write_end()) pos 99287040, len 4096, copied 4096 00000080:00200000:0.0:1775764896.246231:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000230bc732[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.246233:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000000156da1 17ffffc0001001 3:0 ffff90c21282b8c0 24240 lru 00000080:00200000:0.0:1775764896.246237:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000014b06543 24240: 1< 0 - - > 2< 99287040 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1327104 - | + - + - > 5< + - + - | 0 - | 3760 - -> 00000080:00200000:0.0:1775764896.246238:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000230bc732 00000080:00200000:0.0:1775764896.246239:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.246240:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.246245:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.246251:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.246255:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.246263:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.246271:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.246272:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.246273:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.246280:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.246282:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.246283:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.246286:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246291:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.246296:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.246297:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.246300:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.246301:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24240, 24240] 00000080:00200000:0.0:1775764896.246308:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.246310:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.246329:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99291136 i_blocks 90112 00000080:00200000:0.0:1775764896.246332:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.246347:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99291136, count: 4096 00000080:00200000:0.0:1775764896.246356:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99291136, bytes: 4096 00000080:00200000:0.0:1775764896.246359:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.246363:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24242, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.246364:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99291136, 99295232) 0 0 00000080:00200000:0.0:1775764896.246366:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.246371:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24241, 24241] 00020000:00200000:0.0:1775764896.246374:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99291136 chunk: [99291136, 99295232) 99295232, 4096 00020000:00200000:0.0:1775764896.246377:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.246389:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24241, 24241] 00000008:00010000:0.0:1775764896.246394:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.246396:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.246404:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246411:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99291136 99295231) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246417:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246421:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246425:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.246430:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99291136, 99295232) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.246434:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24241 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.246449:0:18302:0:(rw26.c:858:ll_write_end()) pos 99291136, len 4096, copied 4096 00000080:00200000:0.0:1775764896.246453:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d7e4c9d9[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.246456:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cde7d440 17ffffc0001001 3:0 ffff90c21282bb60 24241 lru 00000080:00200000:0.0:1775764896.246461:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000067493b07 24241: 1< 0 - - > 2< 99291136 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1323008 - | + - + - > 5< + - + - | 0 - | 3761 - -> 00000080:00200000:0.0:1775764896.246463:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d7e4c9d9 00000080:00200000:0.0:1775764896.246465:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.246467:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.246475:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.246483:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.246490:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.246498:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.246510:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.246513:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.246515:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.246524:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.246526:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.246529:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.246532:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246540:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.246549:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.246552:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.246556:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.246559:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24241, 24241] 00000080:00200000:0.0:1775764896.246568:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.246571:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.246599:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99295232 i_blocks 90112 00000080:00200000:0.0:1775764896.246603:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.246619:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99295232, count: 4096 00000080:00200000:0.0:1775764896.246630:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99295232, bytes: 4096 00000080:00200000:0.0:1775764896.246633:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.246636:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24243, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.246638:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99295232, 99299328) 0 0 00000080:00200000:0.0:1775764896.246640:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.246645:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24242, 24242] 00020000:00200000:0.0:1775764896.246648:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99295232 chunk: [99295232, 99299328) 99299328, 4096 00020000:00200000:0.0:1775764896.246650:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.246661:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24242, 24242] 00000008:00010000:0.0:1775764896.246666:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.246668:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.246675:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246684:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99295232 99299327) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246693:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246700:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246712:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.246719:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99295232, 99299328) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.246724:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24242 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.246751:0:18302:0:(rw26.c:858:ll_write_end()) pos 99295232, len 4096, copied 4096 00000080:00200000:0.0:1775764896.246755:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f78de300[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.246758:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000d1d885e 17ffffc0001001 3:0 ffff90c21282c9a0 24242 lru 00000080:00200000:0.0:1775764896.246763:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a6cd2bdb 24242: 1< 0 - - > 2< 99295232 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1318912 - | + - + - > 5< + - + - | 0 - | 3762 - -> 00000080:00200000:0.0:1775764896.246765:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f78de300 00000080:00200000:0.0:1775764896.246766:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.246768:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.246776:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.246784:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.246791:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.246798:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.246809:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.246812:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.246815:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.246824:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.246827:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.246829:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.246833:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.246840:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.246848:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.246850:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.246854:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.246857:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24242, 24242] 00000080:00200000:0.0:1775764896.246867:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.246944:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.246976:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99299328 i_blocks 90112 00000080:00200000:0.0:1775764896.246980:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.246997:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99299328, count: 4096 00000080:00200000:0.0:1775764896.247009:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99299328, bytes: 4096 00000080:00200000:0.0:1775764896.247012:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.247015:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24244, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.247017:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99299328, 99303424) 0 0 00000080:00200000:0.0:1775764896.247019:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.247025:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24243, 24243] 00020000:00200000:0.0:1775764896.247028:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99299328 chunk: [99299328, 99303424) 99303424, 4096 00020000:00200000:0.0:1775764896.247030:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.247042:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24243, 24243] 00000008:00010000:0.0:1775764896.247048:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.247051:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.247059:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247068:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99299328 99303423) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247078:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247085:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247091:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.247099:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99299328, 99303424) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.247104:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24243 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.247131:0:18302:0:(rw26.c:858:ll_write_end()) pos 99299328, len 4096, copied 4096 00000080:00200000:0.0:1775764896.247136:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008901af66[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.247138:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007c381bef 17ffffc0001001 3:0 ffff90c21282c2a0 24243 lru 00000080:00200000:0.0:1775764896.247144:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007770d2aa 24243: 1< 0 - - > 2< 99299328 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1314816 - | + - + - > 5< + - + - | 0 - | 3763 - -> 00000080:00200000:0.0:1775764896.247146:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008901af66 00000080:00200000:0.0:1775764896.247147:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.247149:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.247157:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.247165:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.247172:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.247180:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.247192:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.247195:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.247197:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.247207:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.247210:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.247213:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.247217:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247225:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.247232:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.247234:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.247238:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.247241:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24243, 24243] 00000080:00200000:0.0:1775764896.247252:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.247255:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.247275:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99303424 i_blocks 90112 00000080:00200000:0.0:1775764896.247278:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.247289:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99303424, count: 4096 00000080:00200000:0.0:1775764896.247295:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99303424, bytes: 4096 00000080:00200000:0.0:1775764896.247297:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.247299:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24245, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.247300:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99303424, 99307520) 0 0 00000080:00200000:0.0:1775764896.247302:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.247305:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24244, 24244] 00020000:00200000:0.0:1775764896.247308:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99303424 chunk: [99303424, 99307520) 99307520, 4096 00020000:00200000:0.0:1775764896.247309:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.247316:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24244, 24244] 00000008:00010000:0.0:1775764896.247319:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.247321:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.247327:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247336:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99303424 99307519) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247347:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247354:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247361:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.247368:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99303424, 99307520) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.247374:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24244 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.247391:0:18302:0:(rw26.c:858:ll_write_end()) pos 99303424, len 4096, copied 4096 00000080:00200000:0.0:1775764896.247395:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000065547f0[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.247398:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cec1761e 17ffffc0001001 3:0 ffff90c21282cd20 24244 lru 00000080:00200000:0.0:1775764896.247404:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000e28e5bd 24244: 1< 0 - - > 2< 99303424 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1310720 - | + - + - > 5< + - + - | 0 - | 3764 - -> 00000080:00200000:0.0:1775764896.247405:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000065547f0 00000080:00200000:0.0:1775764896.247406:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.247408:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.247414:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.247419:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.247423:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.247427:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.247434:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.247436:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.247437:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.247446:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.247448:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.247450:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.247454:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247461:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.247469:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.247471:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.247475:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.247478:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24244, 24244] 00000080:00200000:0.0:1775764896.247488:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.247492:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.247521:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99307520 i_blocks 90112 00000080:00200000:0.0:1775764896.247525:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.247540:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99307520, count: 4096 00000080:00200000:0.0:1775764896.247550:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99307520, bytes: 4096 00000080:00200000:0.0:1775764896.247553:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.247557:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24246, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.247558:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99307520, 99311616) 0 0 00000080:00200000:0.0:1775764896.247561:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.247566:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24245, 24245] 00020000:00200000:0.0:1775764896.247569:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99307520 chunk: [99307520, 99311616) 99311616, 4096 00020000:00200000:0.0:1775764896.247571:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.247582:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24245, 24245] 00000008:00010000:0.0:1775764896.247586:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.247589:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.247596:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247605:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99307520 99311615) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247613:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247626:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247633:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.247641:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99307520, 99311616) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.247646:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24245 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.247663:0:18302:0:(rw26.c:858:ll_write_end()) pos 99307520, len 4096, copied 4096 00000080:00200000:0.0:1775764896.247668:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ae1968b2[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.247670:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d01c5c6e 17ffffc0001001 3:0 ffff90c21282c380 24245 lru 00000080:00200000:0.0:1775764896.247676:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c1ec2e99 24245: 1< 0 - - > 2< 99307520 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1306624 - | + - + - > 5< + - + - | 0 - | 3765 - -> 00000080:00200000:0.0:1775764896.247678:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ae1968b2 00000080:00200000:0.0:1775764896.247679:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.247681:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.247689:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.247696:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.247703:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.247711:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.247727:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.247730:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.247732:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.247741:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.247743:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.247746:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.247751:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247758:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.247767:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.247769:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.247774:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.247776:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24245, 24245] 00000080:00200000:0.0:1775764896.247786:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.247789:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.247817:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99311616 i_blocks 90112 00000080:00200000:0.0:1775764896.247821:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.247837:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99311616, count: 4096 00000080:00200000:0.0:1775764896.247847:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99311616, bytes: 4096 00000080:00200000:0.0:1775764896.247850:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.247854:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24247, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.247855:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99311616, 99315712) 0 0 00000080:00200000:0.0:1775764896.247857:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.247862:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24246, 24246] 00020000:00200000:0.0:1775764896.247865:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99311616 chunk: [99311616, 99315712) 99315712, 4096 00020000:00200000:0.0:1775764896.247867:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.247959:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24246, 24246] 00000008:00010000:0.0:1775764896.247965:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.247967:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.247975:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247985:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99311616 99315711) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.247993:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248001:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248007:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.248016:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99311616, 99315712) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.248021:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24246 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.248037:0:18302:0:(rw26.c:858:ll_write_end()) pos 99311616, len 4096, copied 4096 00000080:00200000:0.0:1775764896.248041:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f21901fc[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.248044:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f98975bd 17ffffc0001001 3:0 ffff90c21282c620 24246 lru 00000080:00200000:0.0:1775764896.248049:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000033f73349 24246: 1< 0 - - > 2< 99311616 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1302528 - | + - + - > 5< + - + - | 0 - | 3766 - -> 00000080:00200000:0.0:1775764896.248051:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f21901fc 00000080:00200000:0.0:1775764896.248052:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.248055:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.248063:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248071:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248078:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248086:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.248096:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.248098:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.248100:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.248107:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.248109:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.248111:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.248125:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248132:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.248138:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.248139:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.248142:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.248144:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24246, 24246] 00000080:00200000:0.0:1775764896.248152:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.248154:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.248173:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99315712 i_blocks 90112 00000080:00200000:0.0:1775764896.248176:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.248187:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99315712, count: 4096 00000080:00200000:0.0:1775764896.248193:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99315712, bytes: 4096 00000080:00200000:0.0:1775764896.248194:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.248197:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24248, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.248198:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99315712, 99319808) 0 0 00000080:00200000:0.0:1775764896.248200:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.248203:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24247, 24247] 00020000:00200000:0.0:1775764896.248205:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99315712 chunk: [99315712, 99319808) 99319808, 4096 00020000:00200000:0.0:1775764896.248207:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.248213:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24247, 24247] 00000008:00010000:0.0:1775764896.248216:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.248218:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.248223:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248229:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99315712 99319807) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248235:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248239:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248243:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.248248:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99315712, 99319808) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.248252:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24247 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.248264:0:18302:0:(rw26.c:858:ll_write_end()) pos 99315712, len 4096, copied 4096 00000080:00200000:0.0:1775764896.248266:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000001693fbf[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.248268:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000023db3139 17ffffc0001001 3:0 ffff90c21282ce00 24247 lru 00000080:00200000:0.0:1775764896.248272:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c65be4e8 24247: 1< 0 - - > 2< 99315712 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1298432 - | + - + - > 5< + - + - | 0 - | 3767 - -> 00000080:00200000:0.0:1775764896.248273:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000001693fbf 00000080:00200000:0.0:1775764896.248274:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.248276:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.248280:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248285:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248289:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248293:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.248300:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.248301:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.248303:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.248309:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.248310:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.248312:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.248315:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248320:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.248325:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.248327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.248331:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.248333:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24247, 24247] 00000080:00200000:0.0:1775764896.248343:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.248345:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.248374:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99319808 i_blocks 90112 00000080:00200000:0.0:1775764896.248378:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.248394:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99319808, count: 4096 00000080:00200000:0.0:1775764896.248405:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99319808, bytes: 4096 00000080:00200000:0.0:1775764896.248408:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.248411:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24249, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.248412:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99319808, 99323904) 0 0 00000080:00200000:0.0:1775764896.248413:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.248417:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24248, 24248] 00020000:00200000:0.0:1775764896.248419:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99319808 chunk: [99319808, 99323904) 99323904, 4096 00020000:00200000:0.0:1775764896.248421:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.248428:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24248, 24248] 00000008:00010000:0.0:1775764896.248431:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.248433:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.248439:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248448:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99319808 99323903) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248457:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248463:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248469:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.248477:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99319808, 99323904) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.248482:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24248 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.248498:0:18302:0:(rw26.c:858:ll_write_end()) pos 99319808, len 4096, copied 4096 00000080:00200000:0.0:1775764896.248503:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000156d0293[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.248505:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000058a63144 17ffffc0001001 3:0 ffff90c21282c540 24248 lru 00000080:00200000:0.0:1775764896.248511:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002da0365c 24248: 1< 0 - - > 2< 99319808 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1294336 - | + - + - > 5< + - + - | 0 - | 3768 - -> 00000080:00200000:0.0:1775764896.248513:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000156d0293 00000080:00200000:0.0:1775764896.248514:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.248516:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.248525:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248532:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248539:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248552:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.248564:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.248566:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.248569:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.248578:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.248580:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.248583:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.248587:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248594:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.248603:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.248605:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.248609:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.248612:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24248, 24248] 00000080:00200000:0.0:1775764896.248622:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.248626:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.248654:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99323904 i_blocks 90112 00000080:00200000:0.0:1775764896.248659:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.248674:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99323904, count: 4096 00000080:00200000:0.0:1775764896.248684:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99323904, bytes: 4096 00000080:00200000:0.0:1775764896.248687:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.248690:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24250, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.248692:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99323904, 99328000) 0 0 00000080:00200000:0.0:1775764896.248695:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.248700:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24249, 24249] 00020000:00200000:0.0:1775764896.248703:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99323904 chunk: [99323904, 99328000) 99328000, 4096 00020000:00200000:0.0:1775764896.248705:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.248716:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24249, 24249] 00000008:00010000:0.0:1775764896.248721:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.248724:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.248731:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248739:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99323904 99327999) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248747:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248755:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248763:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.248770:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99323904, 99328000) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.248775:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24249 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.248791:0:18302:0:(rw26.c:858:ll_write_end()) pos 99323904, len 4096, copied 4096 00000080:00200000:0.0:1775764896.248795:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b97978b6[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.248797:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b8ced40c 17ffffc0001001 3:0 ffff90c21282c460 24249 lru 00000080:00200000:0.0:1775764896.248803:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000567e2455 24249: 1< 0 - - > 2< 99323904 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1290240 - | + - + - > 5< + - + - | 0 - | 3769 - -> 00000080:00200000:0.0:1775764896.248805:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b97978b6 00000080:00200000:0.0:1775764896.248806:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.248808:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.248815:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248823:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248831:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.248838:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.248849:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.248852:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.248855:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.248863:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.248866:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.248890:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.248894:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.248902:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.248911:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.248914:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.248918:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.248921:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24249, 24249] 00000080:00200000:0.0:1775764896.248931:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.248935:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.248963:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99328000 i_blocks 90112 00000080:00200000:0.0:1775764896.248967:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.248983:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99328000, count: 4096 00000080:00200000:0.0:1775764896.248994:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99328000, bytes: 4096 00000080:00200000:0.0:1775764896.248997:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.249000:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24251, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.249002:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99328000, 99332096) 0 0 00000080:00200000:0.0:1775764896.249004:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.249009:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24250, 24250] 00020000:00200000:0.0:1775764896.249012:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99328000 chunk: [99328000, 99332096) 99332096, 4096 00020000:00200000:0.0:1775764896.249014:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.249026:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24250, 24250] 00000008:00010000:0.0:1775764896.249030:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.249033:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.249041:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249050:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99328000 99332095) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249058:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249066:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249076:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.249082:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99328000, 99332096) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.249085:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24250 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.249098:0:18302:0:(rw26.c:858:ll_write_end()) pos 99328000, len 4096, copied 4096 00000080:00200000:0.0:1775764896.249101:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001eb360e1[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.249102:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000095410294 17ffffc0001001 3:0 ffff90c21282ca80 24250 lru 00000080:00200000:0.0:1775764896.249106:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ff145069 24250: 1< 0 - - > 2< 99328000 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1286144 - | + - + - > 5< + - + - | 0 - | 3770 - -> 00000080:00200000:0.0:1775764896.249107:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001eb360e1 00000080:00200000:0.0:1775764896.249108:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.249110:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.249128:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249134:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249138:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249142:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.249149:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.249151:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.249153:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.249160:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.249162:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.249164:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.249166:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249171:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.249177:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.249178:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.249180:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.249182:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24250, 24250] 00000080:00200000:0.0:1775764896.249189:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.249191:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.249209:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99332096 i_blocks 90112 00000080:00200000:0.0:1775764896.249212:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.249223:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99332096, count: 4096 00000080:00200000:0.0:1775764896.249229:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99332096, bytes: 4096 00000080:00200000:0.0:1775764896.249230:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.249233:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24252, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.249234:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99332096, 99336192) 0 0 00000080:00200000:0.0:1775764896.249235:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.249239:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24251, 24251] 00020000:00200000:0.0:1775764896.249241:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99332096 chunk: [99332096, 99336192) 99336192, 4096 00020000:00200000:0.0:1775764896.249242:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.249248:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24251, 24251] 00000008:00010000:0.0:1775764896.249251:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.249253:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.249259:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249265:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99332096 99336191) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249271:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249275:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249279:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.249283:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99332096, 99336192) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.249286:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24251 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.249299:0:18302:0:(rw26.c:858:ll_write_end()) pos 99332096, len 4096, copied 4096 00000080:00200000:0.0:1775764896.249302:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002ee086e2[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.249303:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009bf42d42 17ffffc0001001 3:0 ffff90c21282c700 24251 lru 00000080:00200000:0.0:1775764896.249307:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000580280b9 24251: 1< 0 - - > 2< 99332096 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1282048 - | + - + - > 5< + - + - | 0 - | 3771 - -> 00000080:00200000:0.0:1775764896.249308:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002ee086e2 00000080:00200000:0.0:1775764896.249308:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.249310:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.249315:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249319:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249324:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249330:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.249341:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.249344:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.249346:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.249354:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.249357:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.249359:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.249363:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249370:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.249378:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.249380:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.249385:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.249387:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24251, 24251] 00000080:00200000:0.0:1775764896.249397:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.249401:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.249423:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99336192 i_blocks 90112 00000080:00200000:0.0:1775764896.249426:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.249437:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99336192, count: 4096 00000080:00200000:0.0:1775764896.249446:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99336192, bytes: 4096 00000080:00200000:0.0:1775764896.249448:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.249452:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24253, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.249453:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99336192, 99340288) 0 0 00000080:00200000:0.0:1775764896.249456:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.249461:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24252, 24252] 00020000:00200000:0.0:1775764896.249464:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99336192 chunk: [99336192, 99340288) 99340288, 4096 00020000:00200000:0.0:1775764896.249466:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.249477:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24252, 24252] 00000008:00010000:0.0:1775764896.249482:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.249484:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.249492:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249500:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99336192 99340287) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249509:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249517:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249523:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.249530:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99336192, 99340288) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.249535:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24252 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.249557:0:18302:0:(rw26.c:858:ll_write_end()) pos 99336192, len 4096, copied 4096 00000080:00200000:0.0:1775764896.249561:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a3e22f89[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.249564:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000056d53eee 17ffffc0001021 2:0 ffff90c21282cc40 24252 lru 00000080:00200000:0.0:1775764896.249570:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000408d628d 24252: 1< 0 - - > 2< 99336192 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1277952 - | + - + - > 5< + - + - | 0 - | 3772 - -> 00000080:00200000:0.0:1775764896.249572:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a3e22f89 00000080:00200000:0.0:1775764896.249573:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.249575:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.249583:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249590:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249596:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249604:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.249615:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.249618:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.249620:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.249629:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.249631:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.249634:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.249638:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249646:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.249654:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.249656:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.249660:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.249663:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24252, 24252] 00000080:00200000:0.0:1775764896.249673:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.249676:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.249704:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99340288 i_blocks 90112 00000080:00200000:0.0:1775764896.249708:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.249723:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99340288, count: 4096 00000080:00200000:0.0:1775764896.249733:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99340288, bytes: 4096 00000080:00200000:0.0:1775764896.249736:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.249740:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24254, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.249741:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99340288, 99344384) 0 0 00000080:00200000:0.0:1775764896.249743:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.249749:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24253, 24253] 00020000:00200000:0.0:1775764896.249752:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99340288 chunk: [99340288, 99344384) 99344384, 4096 00020000:00200000:0.0:1775764896.249754:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.249765:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24253, 24253] 00000008:00010000:0.0:1775764896.249770:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.249773:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.249780:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249789:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99340288 99344383) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249796:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249808:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.249815:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.249823:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99340288, 99344384) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.249828:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24253 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.249844:0:18302:0:(rw26.c:858:ll_write_end()) pos 99340288, len 4096, copied 4096 00000080:00200000:0.0:1775764896.249848:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000dcd36913[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.249851:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000384e2da3 17ffffc0001001 3:0 ffff90c21282c8c0 24253 lru 00000080:00200000:0.0:1775764896.249856:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002a327cb7 24253: 1< 0 - - > 2< 99340288 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1273856 - | + - + - > 5< + - + - | 0 - | 3773 - -> 00000080:00200000:0.0:1775764896.249858:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000dcd36913 00000080:00200000:0.0:1775764896.249859:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.249861:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.249956:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249965:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249972:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.249980:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.249998:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.250001:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.250003:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.250014:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.250016:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.250018:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.250022:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250030:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.250038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.250040:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.250044:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.250047:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24253, 24253] 00000080:00200000:0.0:1775764896.250058:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.250061:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.250084:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99344384 i_blocks 90112 00000080:00200000:0.0:1775764896.250087:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.250097:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99344384, count: 4096 00000080:00200000:0.0:1775764896.250103:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99344384, bytes: 4096 00000080:00200000:0.0:1775764896.250105:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.250108:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24255, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.250109:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99344384, 99348480) 0 0 00000080:00200000:0.0:1775764896.250111:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.250126:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24254, 24254] 00020000:00200000:0.0:1775764896.250129:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99344384 chunk: [99344384, 99348480) 99348480, 4096 00020000:00200000:0.0:1775764896.250130:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.250138:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24254, 24254] 00000008:00010000:0.0:1775764896.250142:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.250143:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.250149:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250155:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99344384 99348479) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250160:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250165:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250169:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.250174:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99344384, 99348480) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.250177:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24254 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.250190:0:18302:0:(rw26.c:858:ll_write_end()) pos 99344384, len 4096, copied 4096 00000080:00200000:0.0:1775764896.250193:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f671f237[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.250194:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006cfb000d 17ffffc0001001 3:0 ffff90c21282cb60 24254 lru 00000080:00200000:0.0:1775764896.250198:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004a331bdc 24254: 1< 0 - - > 2< 99344384 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1269760 - | + - + - > 5< + - + - | 0 - | 3774 - -> 00000080:00200000:0.0:1775764896.250199:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f671f237 00000080:00200000:0.0:1775764896.250200:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.250202:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.250207:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.250212:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.250217:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.250221:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.250227:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.250229:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.250230:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.250237:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.250238:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.250239:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.250242:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250246:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.250251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.250253:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.250255:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.250257:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24254, 24254] 00000080:00200000:0.0:1775764896.250264:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.250266:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.250284:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99348480 i_blocks 90112 00000080:00200000:0.0:1775764896.250287:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.250297:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99348480, count: 4096 00000080:00200000:0.0:1775764896.250303:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99348480, bytes: 4096 00000080:00200000:0.0:1775764896.250305:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.250307:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24256, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.250308:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99348480, 99352576) 0 0 00000080:00200000:0.0:1775764896.250310:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.250313:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24255, 24255] 00020000:00200000:0.0:1775764896.250315:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99348480 chunk: [99348480, 99352576) 99352576, 4096 00020000:00200000:0.0:1775764896.250316:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.250323:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24255, 24255] 00000008:00010000:0.0:1775764896.250327:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.250329:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.250336:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250344:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99348480 99352575) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250352:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250359:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250366:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.250375:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99348480, 99352576) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.250380:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24255 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.250395:0:18302:0:(rw26.c:858:ll_write_end()) pos 99348480, len 4096, copied 4096 00000080:00200000:0.0:1775764896.250400:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ebdf836d[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.250402:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005423cc0d 17ffffc0001001 3:0 ffff90c21282c000 24255 lru 00000080:00200000:0.0:1775764896.250407:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cc204f10 24255: 1< 0 - - > 2< 99348480 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1265664 - | + - + - > 5< + - + - | 0 - | 3775 - -> 00000080:00200000:0.0:1775764896.250408:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ebdf836d 00000080:00200000:0.0:1775764896.250409:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.250411:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.250416:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.250421:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.250425:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.250429:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.250436:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.250439:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.250441:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.250449:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.250452:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.250454:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.250457:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250465:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.250473:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.250475:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.250479:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.250482:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24255, 24255] 00000080:00200000:0.0:1775764896.250492:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.250495:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.250524:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99352576 i_blocks 90112 00000080:00200000:0.0:1775764896.250527:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.250542:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99352576, count: 4096 00000080:00200000:0.0:1775764896.250553:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99352576, bytes: 4096 00000080:00200000:0.0:1775764896.250556:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.250559:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24257, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.250561:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99352576, 99356672) 0 0 00000080:00200000:0.0:1775764896.250563:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.250568:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24256, 24256] 00020000:00200000:0.0:1775764896.250571:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99352576 chunk: [99352576, 99356672) 99356672, 4096 00020000:00200000:0.0:1775764896.250574:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.250585:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24256, 24256] 00000008:00010000:0.0:1775764896.250590:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.250593:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.250600:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250609:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99352576 99356671) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250618:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250626:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250633:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.250641:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99352576, 99356672) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.250646:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24256 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.250668:0:18302:0:(rw26.c:858:ll_write_end()) pos 99352576, len 4096, copied 4096 00000080:00200000:0.0:1775764896.250672:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d6cf51bb[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.250675:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008d610017 17ffffc0001001 3:0 ffff90c21282cee0 24256 lru 00000080:00200000:0.0:1775764896.250680:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f012e911 24256: 1< 0 - - > 2< 99352576 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1261568 - | + - + - > 5< + - + - | 0 - | 3776 - -> 00000080:00200000:0.0:1775764896.250683:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d6cf51bb 00000080:00200000:0.0:1775764896.250684:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.250686:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.250693:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.250702:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.250709:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.250721:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.250732:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.250735:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.250737:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.250746:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.250749:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.250751:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.250755:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250764:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.250772:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.250774:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.250778:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.250781:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24256, 24256] 00000080:00200000:0.0:1775764896.250790:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.250793:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.250822:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99356672 i_blocks 90112 00000080:00200000:0.0:1775764896.250826:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.250841:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99356672, count: 4096 00000080:00200000:0.0:1775764896.250853:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99356672, bytes: 4096 00000080:00200000:0.0:1775764896.250855:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.250859:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24258, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.250861:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99356672, 99360768) 0 0 00000080:00200000:0.0:1775764896.250863:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.250950:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24257, 24257] 00020000:00200000:0.0:1775764896.250954:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99356672 chunk: [99356672, 99360768) 99360768, 4096 00020000:00200000:0.0:1775764896.250957:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.250971:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24257, 24257] 00000008:00010000:0.0:1775764896.250976:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.250978:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.250986:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.250995:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99356672 99360767) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251003:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251010:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251018:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.251025:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99356672, 99360768) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.251030:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24257 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.251047:0:18302:0:(rw26.c:858:ll_write_end()) pos 99356672, len 4096, copied 4096 00000080:00200000:0.0:1775764896.251052:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009017d692[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.251055:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f9c5a595 17ffffc0001001 3:0 ffff90c21282c0e0 24257 lru 00000080:00200000:0.0:1775764896.251060:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a2e0fe6e 24257: 1< 0 - - > 2< 99356672 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1257472 - | + - + - > 5< + - + - | 0 - | 3777 - -> 00000080:00200000:0.0:1775764896.251062:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009017d692 00000080:00200000:0.0:1775764896.251063:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.251066:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.251073:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251080:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251087:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251094:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.251105:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.251108:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.251110:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.251130:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.251133:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.251135:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.251139:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251147:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.251155:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.251158:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.251162:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.251166:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24257, 24257] 00000080:00200000:0.0:1775764896.251174:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.251177:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.251196:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99360768 i_blocks 90112 00000080:00200000:0.0:1775764896.251199:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.251210:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99360768, count: 4096 00000080:00200000:0.0:1775764896.251216:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99360768, bytes: 4096 00000080:00200000:0.0:1775764896.251218:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.251220:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24259, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.251221:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99360768, 99364864) 0 0 00000080:00200000:0.0:1775764896.251223:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.251226:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24258, 24258] 00020000:00200000:0.0:1775764896.251228:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99360768 chunk: [99360768, 99364864) 99364864, 4096 00020000:00200000:0.0:1775764896.251230:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.251237:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24258, 24258] 00000008:00010000:0.0:1775764896.251240:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.251242:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.251247:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251253:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99360768 99364863) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251258:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251263:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251271:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.251275:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99360768, 99364864) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.251278:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24258 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.251293:0:18302:0:(rw26.c:858:ll_write_end()) pos 99360768, len 4096, copied 4096 00000080:00200000:0.0:1775764896.251296:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000f657cd3[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.251298:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f0f3d17f 17ffffc0001001 3:0 ffff90c21282c1c0 24258 lru 00000080:00200000:0.0:1775764896.251301:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000014f2eac8 24258: 1< 0 - - > 2< 99360768 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1253376 - | + - + - > 5< + - + - | 0 - | 3778 - -> 00000080:00200000:0.0:1775764896.251303:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000f657cd3 00000080:00200000:0.0:1775764896.251303:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.251305:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.251310:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251315:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251319:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251323:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.251332:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.251334:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.251337:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.251345:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.251348:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.251350:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.251354:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251361:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.251369:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.251372:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.251376:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.251378:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24258, 24258] 00000080:00200000:0.0:1775764896.251389:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.251392:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.251417:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99364864 i_blocks 90112 00000080:00200000:0.0:1775764896.251420:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.251430:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99364864, count: 4096 00000080:00200000:0.0:1775764896.251437:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99364864, bytes: 4096 00000080:00200000:0.0:1775764896.251439:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.251443:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24260, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.251444:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99364864, 99368960) 0 0 00000080:00200000:0.0:1775764896.251446:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.251451:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24259, 24259] 00020000:00200000:0.0:1775764896.251454:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99364864 chunk: [99364864, 99368960) 99368960, 4096 00020000:00200000:0.0:1775764896.251456:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.251467:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24259, 24259] 00000008:00010000:0.0:1775764896.251472:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.251475:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.251483:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251491:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99364864 99368959) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251500:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251508:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251515:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.251522:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99364864, 99368960) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.251527:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24259 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.251543:0:18302:0:(rw26.c:858:ll_write_end()) pos 99364864, len 4096, copied 4096 00000080:00200000:0.0:1775764896.251547:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003f1e73cd[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.251550:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ff8a54af 17ffffc0001001 3:0 ffff90c21282c7e0 24259 lru 00000080:00200000:0.0:1775764896.251555:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f3c199c6 24259: 1< 0 - - > 2< 99364864 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1249280 - | + - + - > 5< + - + - | 0 - | 3779 - -> 00000080:00200000:0.0:1775764896.251557:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003f1e73cd 00000080:00200000:0.0:1775764896.251558:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.251561:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.251568:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251576:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251583:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251589:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.251600:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.251603:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.251605:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.251614:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.251616:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.251619:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.251623:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251630:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.251639:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.251641:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.251645:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.251648:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24259, 24259] 00000080:00200000:0.0:1775764896.251658:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.251661:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.251689:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99368960 i_blocks 90112 00000080:00200000:0.0:1775764896.251692:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.251709:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99368960, count: 4096 00000080:00200000:0.0:1775764896.251719:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99368960, bytes: 4096 00000080:00200000:0.0:1775764896.251722:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.251725:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24261, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.251727:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99368960, 99373056) 0 0 00000080:00200000:0.0:1775764896.251729:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.251734:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24260, 24260] 00020000:00200000:0.0:1775764896.251737:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99368960 chunk: [99368960, 99373056) 99373056, 4096 00020000:00200000:0.0:1775764896.251739:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.251750:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24260, 24260] 00000008:00010000:0.0:1775764896.251755:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.251758:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.251765:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251774:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99368960 99373055) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251783:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251790:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.251797:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.251805:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99368960, 99373056) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.251810:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24260 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.251837:0:18302:0:(rw26.c:858:ll_write_end()) pos 99368960, len 4096, copied 4096 00000080:00200000:0.0:1775764896.251841:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000509dd50d[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.251844:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000270c14a 17ffffc0001001 3:0 ffff90c21282d7e0 24260 lru 00000080:00200000:0.0:1775764896.251850:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000965d9d79 24260: 1< 0 - - > 2< 99368960 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1245184 - | + - + - > 5< + - + - | 0 - | 3780 - -> 00000080:00200000:0.0:1775764896.251852:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000509dd50d 00000080:00200000:0.0:1775764896.251853:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.251855:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.251863:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251949:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251957:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.251965:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.251978:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.251981:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.251983:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.251994:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.251996:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.251999:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.252003:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252011:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.252019:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.252021:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.252025:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.252028:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24260, 24260] 00000080:00200000:0.0:1775764896.252038:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.252041:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.252071:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99373056 i_blocks 90112 00000080:00200000:0.0:1775764896.252075:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.252090:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99373056, count: 4096 00000080:00200000:0.0:1775764896.252098:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99373056, bytes: 4096 00000080:00200000:0.0:1775764896.252100:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.252102:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24262, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.252103:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99373056, 99377152) 0 0 00000080:00200000:0.0:1775764896.252105:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.252109:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24261, 24261] 00020000:00200000:0.0:1775764896.252111:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99373056 chunk: [99373056, 99377152) 99377152, 4096 00020000:00200000:0.0:1775764896.252124:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.252134:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24261, 24261] 00000008:00010000:0.0:1775764896.252138:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.252140:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.252146:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252152:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99373056 99377151) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252157:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252165:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252170:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.252174:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99373056, 99377152) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.252177:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24261 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.252190:0:18302:0:(rw26.c:858:ll_write_end()) pos 99373056, len 4096, copied 4096 00000080:00200000:0.0:1775764896.252192:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e91845e1[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.252194:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000080fce0ea 17ffffc0001001 3:0 ffff90c21282d9a0 24261 lru 00000080:00200000:0.0:1775764896.252197:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005736b609 24261: 1< 0 - - > 2< 99373056 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1241088 - | + - + - > 5< + - + - | 0 - | 3781 - -> 00000080:00200000:0.0:1775764896.252198:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e91845e1 00000080:00200000:0.0:1775764896.252199:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.252200:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.252205:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.252210:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.252214:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.252218:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.252230:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.252231:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.252233:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.252239:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.252241:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.252243:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.252245:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252251:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.252256:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.252257:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.252260:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.252261:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24261, 24261] 00000080:00200000:0.0:1775764896.252268:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.252270:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.252288:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99377152 i_blocks 90112 00000080:00200000:0.0:1775764896.252291:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.252302:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99377152, count: 4096 00000080:00200000:0.0:1775764896.252307:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99377152, bytes: 4096 00000080:00200000:0.0:1775764896.252309:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.252312:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24263, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.252313:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99377152, 99381248) 0 0 00000080:00200000:0.0:1775764896.252314:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.252318:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24262, 24262] 00020000:00200000:0.0:1775764896.252320:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99377152 chunk: [99377152, 99381248) 99381248, 4096 00020000:00200000:0.0:1775764896.252321:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.252329:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24262, 24262] 00000008:00010000:0.0:1775764896.252334:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.252337:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.252344:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252351:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99377152 99381247) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252359:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252366:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252373:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.252381:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99377152, 99381248) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.252386:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24262 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.252402:0:18302:0:(rw26.c:858:ll_write_end()) pos 99377152, len 4096, copied 4096 00000080:00200000:0.0:1775764896.252406:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007e515b7d[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.252409:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007b4c881f 17ffffc0001001 3:0 ffff90c21282d2a0 24262 lru 00000080:00200000:0.0:1775764896.252412:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000095d32293 24262: 1< 0 - - > 2< 99377152 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1236992 - | + - + - > 5< + - + - | 0 - | 3782 - -> 00000080:00200000:0.0:1775764896.252414:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007e515b7d 00000080:00200000:0.0:1775764896.252415:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.252416:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.252422:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.252427:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.252431:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.252435:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.252444:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.252447:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.252449:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.252457:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.252459:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.252462:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.252465:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252472:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.252480:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.252482:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.252487:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.252489:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24262, 24262] 00000080:00200000:0.0:1775764896.252499:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.252503:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.252531:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99381248 i_blocks 90112 00000080:00200000:0.0:1775764896.252535:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.252551:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99381248, count: 4096 00000080:00200000:0.0:1775764896.252561:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99381248, bytes: 4096 00000080:00200000:0.0:1775764896.252564:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.252568:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24264, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.252569:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99381248, 99385344) 0 0 00000080:00200000:0.0:1775764896.252572:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.252577:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24263, 24263] 00020000:00200000:0.0:1775764896.252580:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99381248 chunk: [99381248, 99385344) 99385344, 4096 00020000:00200000:0.0:1775764896.252582:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.252593:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24263, 24263] 00000008:00010000:0.0:1775764896.252598:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.252601:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.252608:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252617:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99381248 99385343) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252625:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252633:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252640:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.252648:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99381248, 99385344) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.252654:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24263 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.252671:0:18302:0:(rw26.c:858:ll_write_end()) pos 99381248, len 4096, copied 4096 00000080:00200000:0.0:1775764896.252675:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000238f891b[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.252678:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f0f87c62 17ffffc0001001 3:0 ffff90c21282dd20 24263 lru 00000080:00200000:0.0:1775764896.252683:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000eb7bc13e 24263: 1< 0 - - > 2< 99381248 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1232896 - | + - + - > 5< + - + - | 0 - | 3783 - -> 00000080:00200000:0.0:1775764896.252685:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000238f891b 00000080:00200000:0.0:1775764896.252686:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.252688:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.252696:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.252704:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.252712:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.252719:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.252730:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.252733:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.252735:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.252744:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.252746:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.252749:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.252754:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252762:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.252771:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.252773:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.252778:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.252780:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24263, 24263] 00000080:00200000:0.0:1775764896.252790:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.252793:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.252821:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99385344 i_blocks 90112 00000080:00200000:0.0:1775764896.252824:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.252841:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99385344, count: 4096 00000080:00200000:0.0:1775764896.252851:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99385344, bytes: 4096 00000080:00200000:0.0:1775764896.252854:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.252857:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24265, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.252859:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99385344, 99389440) 0 0 00000080:00200000:0.0:1775764896.252861:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.252865:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24264, 24264] 00020000:00200000:0.0:1775764896.252888:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99385344 chunk: [99385344, 99389440) 99389440, 4096 00020000:00200000:0.0:1775764896.252891:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.252904:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24264, 24264] 00000008:00010000:0.0:1775764896.252910:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.252912:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.252919:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252928:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99385344 99389439) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252937:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252944:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.252950:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.252958:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99385344, 99389440) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.252963:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24264 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.252979:0:18302:0:(rw26.c:858:ll_write_end()) pos 99385344, len 4096, copied 4096 00000080:00200000:0.0:1775764896.252983:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009b7ecd90[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.252986:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000cacb12e 17ffffc0001001 3:0 ffff90c21282d380 24264 lru 00000080:00200000:0.0:1775764896.252991:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b3ce7aee 24264: 1< 0 - - > 2< 99385344 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1228800 - | + - + - > 5< + - + - | 0 - | 3784 - -> 00000080:00200000:0.0:1775764896.252994:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009b7ecd90 00000080:00200000:0.0:1775764896.252995:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.252997:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.253005:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253013:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253020:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253031:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.253043:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.253046:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.253049:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.253059:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.253061:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.253064:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.253068:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253076:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.253084:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.253087:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.253091:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.253093:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24264, 24264] 00000080:00200000:0.0:1775764896.253100:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.253103:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.253143:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99389440 i_blocks 90112 00000080:00200000:0.0:1775764896.253147:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.253162:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99389440, count: 4096 00000080:00200000:0.0:1775764896.253174:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99389440, bytes: 4096 00000080:00200000:0.0:1775764896.253176:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.253179:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24266, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.253181:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99389440, 99393536) 0 0 00000080:00200000:0.0:1775764896.253183:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.253188:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24265, 24265] 00020000:00200000:0.0:1775764896.253190:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99389440 chunk: [99389440, 99393536) 99393536, 4096 00020000:00200000:0.0:1775764896.253192:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.253202:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24265, 24265] 00000008:00010000:0.0:1775764896.253207:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.253210:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.253218:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253227:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99389440 99393535) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253235:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253241:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253247:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.253254:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99389440, 99393536) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.253259:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24265 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.253275:0:18302:0:(rw26.c:858:ll_write_end()) pos 99389440, len 4096, copied 4096 00000080:00200000:0.0:1775764896.253279:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000e3d7f09[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.253282:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f29f93ac 17ffffc0001001 3:0 ffff90c21282d620 24265 lru 00000080:00200000:0.0:1775764896.253286:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a61fb1f4 24265: 1< 0 - - > 2< 99389440 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1224704 - | + - + - > 5< + - + - | 0 - | 3785 - -> 00000080:00200000:0.0:1775764896.253288:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000e3d7f09 00000080:00200000:0.0:1775764896.253289:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.253291:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.253299:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253305:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253311:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253317:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.253327:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.253330:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.253332:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.253342:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.253344:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.253347:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.253350:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253357:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.253364:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.253367:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.253372:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.253374:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24265, 24265] 00000080:00200000:0.0:1775764896.253384:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.253387:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.253413:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99393536 i_blocks 90112 00000080:00200000:0.0:1775764896.253416:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.253430:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99393536, count: 4096 00000080:00200000:0.0:1775764896.253440:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99393536, bytes: 4096 00000080:00200000:0.0:1775764896.253443:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.253446:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24267, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.253448:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99393536, 99397632) 0 0 00000080:00200000:0.0:1775764896.253450:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.253455:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24266, 24266] 00020000:00200000:0.0:1775764896.253458:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99393536 chunk: [99393536, 99397632) 99397632, 4096 00020000:00200000:0.0:1775764896.253460:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.253471:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24266, 24266] 00000008:00010000:0.0:1775764896.253475:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.253477:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.253485:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253494:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99393536 99397631) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253501:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253508:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253519:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.253526:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99393536, 99397632) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.253531:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24266 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.253549:0:18302:0:(rw26.c:858:ll_write_end()) pos 99393536, len 4096, copied 4096 00000080:00200000:0.0:1775764896.253553:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000665de4ee[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.253555:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000023dc17f7 17ffffc0001001 3:0 ffff90c21282de00 24266 lru 00000080:00200000:0.0:1775764896.253560:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f959e420 24266: 1< 0 - - > 2< 99393536 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1220608 - | + - + - > 5< + - + - | 0 - | 3786 - -> 00000080:00200000:0.0:1775764896.253561:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000665de4ee 00000080:00200000:0.0:1775764896.253562:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.253564:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.253571:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253578:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253585:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253591:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.253602:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.253604:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.253607:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.253616:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.253618:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.253621:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.253625:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253632:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.253639:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.253641:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.253645:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.253648:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24266, 24266] 00000080:00200000:0.0:1775764896.253657:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.253660:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.253689:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99397632 i_blocks 90112 00000080:00200000:0.0:1775764896.253692:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.253708:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99397632, count: 4096 00000080:00200000:0.0:1775764896.253719:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99397632, bytes: 4096 00000080:00200000:0.0:1775764896.253721:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.253725:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24268, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.253727:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99397632, 99401728) 0 0 00000080:00200000:0.0:1775764896.253729:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.253734:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24267, 24267] 00020000:00200000:0.0:1775764896.253737:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99397632 chunk: [99397632, 99401728) 99401728, 4096 00020000:00200000:0.0:1775764896.253739:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.253750:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24267, 24267] 00000008:00010000:0.0:1775764896.253755:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.253758:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.253766:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253775:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99397632 99401727) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253783:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253791:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.253797:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.253805:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99397632, 99401728) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.253809:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24267 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.253831:0:18302:0:(rw26.c:858:ll_write_end()) pos 99397632, len 4096, copied 4096 00000080:00200000:0.0:1775764896.253836:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007368555c[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.253839:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a4a0a556 17ffffc0001021 2:0 ffff90c21282d540 24267 lru 00000080:00200000:0.0:1775764896.253844:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a31cbbbf 24267: 1< 0 - - > 2< 99397632 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1216512 - | + - + - > 5< + - + - | 0 - | 3787 - -> 00000080:00200000:0.0:1775764896.253846:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007368555c 00000080:00200000:0.0:1775764896.253847:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.253849:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.253857:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253864:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253962:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.253971:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.253983:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.253987:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.253989:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.253999:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.254002:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.254004:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.254008:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254016:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.254023:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.254025:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.254029:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.254032:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24267, 24267] 00000080:00200000:0.0:1775764896.254041:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.254044:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.254071:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99401728 i_blocks 90112 00000080:00200000:0.0:1775764896.254075:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.254090:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99401728, count: 4096 00000080:00200000:0.0:1775764896.254100:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99401728, bytes: 4096 00000080:00200000:0.0:1775764896.254103:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.254107:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24269, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.254108:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99401728, 99405824) 0 0 00000080:00200000:0.0:1775764896.254111:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.254127:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24268, 24268] 00020000:00200000:0.0:1775764896.254130:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99401728 chunk: [99401728, 99405824) 99405824, 4096 00020000:00200000:0.0:1775764896.254132:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.254145:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24268, 24268] 00000008:00010000:0.0:1775764896.254150:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.254152:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.254160:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254168:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99401728 99405823) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254176:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254184:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254191:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.254199:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99401728, 99405824) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.254204:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24268 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.254220:0:18302:0:(rw26.c:858:ll_write_end()) pos 99401728, len 4096, copied 4096 00000080:00200000:0.0:1775764896.254225:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002c438cf2[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.254227:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009d8f087e 17ffffc0001001 3:0 ffff90c21282d460 24268 lru 00000080:00200000:0.0:1775764896.254232:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000044611685 24268: 1< 0 - - > 2< 99401728 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1212416 - | + - + - > 5< + - + - | 0 - | 3788 - -> 00000080:00200000:0.0:1775764896.254234:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002c438cf2 00000080:00200000:0.0:1775764896.254235:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.254237:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.254245:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.254253:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.254260:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.254268:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.254279:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.254282:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.254284:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.254293:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.254296:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.254298:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.254302:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254309:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.254317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.254319:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.254323:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.254326:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24268, 24268] 00000080:00200000:0.0:1775764896.254336:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.254339:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.254369:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99405824 i_blocks 90112 00000080:00200000:0.0:1775764896.254373:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.254387:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99405824, count: 4096 00000080:00200000:0.0:1775764896.254398:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99405824, bytes: 4096 00000080:00200000:0.0:1775764896.254400:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.254404:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24270, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.254405:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99405824, 99409920) 0 0 00000080:00200000:0.0:1775764896.254408:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.254413:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24269, 24269] 00020000:00200000:0.0:1775764896.254416:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99405824 chunk: [99405824, 99409920) 99409920, 4096 00020000:00200000:0.0:1775764896.254418:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.254430:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24269, 24269] 00000008:00010000:0.0:1775764896.254434:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.254437:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.254444:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254452:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99405824 99409919) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254460:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254471:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254478:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.254487:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99405824, 99409920) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.254492:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24269 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.254509:0:18302:0:(rw26.c:858:ll_write_end()) pos 99405824, len 4096, copied 4096 00000080:00200000:0.0:1775764896.254513:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000053d0b7a0[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.254516:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ea2d752d 17ffffc0001001 3:0 ffff90c21282da80 24269 lru 00000080:00200000:0.0:1775764896.254521:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ccbad3bc 24269: 1< 0 - - > 2< 99405824 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1208320 - | + - + - > 5< + - + - | 0 - | 3789 - -> 00000080:00200000:0.0:1775764896.254523:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000053d0b7a0 00000080:00200000:0.0:1775764896.254524:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.254525:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.254533:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.254541:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.254547:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.254554:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.254570:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.254572:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.254575:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.254584:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.254586:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.254589:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.254593:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254601:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.254608:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.254610:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.254614:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.254617:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24269, 24269] 00000080:00200000:0.0:1775764896.254627:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.254630:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.254658:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99409920 i_blocks 90112 00000080:00200000:0.0:1775764896.254661:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.254677:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99409920, count: 4096 00000080:00200000:0.0:1775764896.254687:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99409920, bytes: 4096 00000080:00200000:0.0:1775764896.254690:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.254694:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24271, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.254695:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99409920, 99414016) 0 0 00000080:00200000:0.0:1775764896.254697:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.254702:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24270, 24270] 00020000:00200000:0.0:1775764896.254705:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99409920 chunk: [99409920, 99414016) 99414016, 4096 00020000:00200000:0.0:1775764896.254707:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.254718:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24270, 24270] 00000008:00010000:0.0:1775764896.254723:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.254725:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.254733:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254742:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99409920 99414015) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254750:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254757:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254763:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.254770:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99409920, 99414016) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.254775:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24270 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.254790:0:18302:0:(rw26.c:858:ll_write_end()) pos 99409920, len 4096, copied 4096 00000080:00200000:0.0:1775764896.254794:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003e5e557e[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.254797:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000056b74f39 17ffffc0001001 3:0 ffff90c21282d700 24270 lru 00000080:00200000:0.0:1775764896.254802:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a65d59dd 24270: 1< 0 - - > 2< 99409920 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1204224 - | + - + - > 5< + - + - | 0 - | 3790 - -> 00000080:00200000:0.0:1775764896.254804:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003e5e557e 00000080:00200000:0.0:1775764896.254805:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.254807:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.254815:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.254823:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.254830:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.254837:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.254849:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.254852:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.254854:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.254863:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.254865:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.254948:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.254954:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.254961:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.254969:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.254971:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.254976:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.254979:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24270, 24270] 00000080:00200000:0.0:1775764896.254990:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.254993:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.255023:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99414016 i_blocks 90112 00000080:00200000:0.0:1775764896.255027:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.255043:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99414016, count: 4096 00000080:00200000:0.0:1775764896.255054:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99414016, bytes: 4096 00000080:00200000:0.0:1775764896.255056:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.255060:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24272, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.255061:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99414016, 99418112) 0 0 00000080:00200000:0.0:1775764896.255064:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.255069:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24271, 24271] 00020000:00200000:0.0:1775764896.255072:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99414016 chunk: [99414016, 99418112) 99418112, 4096 00020000:00200000:0.0:1775764896.255075:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.255087:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24271, 24271] 00000008:00010000:0.0:1775764896.255092:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.255094:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.255102:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255111:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99414016 99418111) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255131:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255138:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255145:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.255153:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99414016, 99418112) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.255159:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24271 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.255174:0:18302:0:(rw26.c:858:ll_write_end()) pos 99414016, len 4096, copied 4096 00000080:00200000:0.0:1775764896.255178:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b2129c34[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.255181:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a6be96af 17ffffc0001001 3:0 ffff90c21282dc40 24271 lru 00000080:00200000:0.0:1775764896.255187:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d2b2321b 24271: 1< 0 - - > 2< 99414016 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1200128 - | + - + - > 5< + - + - | 0 - | 3791 - -> 00000080:00200000:0.0:1775764896.255190:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b2129c34 00000080:00200000:0.0:1775764896.255191:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.255193:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.255200:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.255208:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.255215:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.255222:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.255233:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.255236:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.255238:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.255248:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.255251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.255253:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.255258:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255266:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.255273:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.255276:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.255280:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.255282:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24271, 24271] 00000080:00200000:0.0:1775764896.255292:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.255295:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.255322:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99418112 i_blocks 90112 00000080:00200000:0.0:1775764896.255326:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.255342:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99418112, count: 4096 00000080:00200000:0.0:1775764896.255351:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99418112, bytes: 4096 00000080:00200000:0.0:1775764896.255354:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.255357:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24273, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.255359:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99418112, 99422208) 0 0 00000080:00200000:0.0:1775764896.255361:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.255367:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24272, 24272] 00020000:00200000:0.0:1775764896.255370:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99418112 chunk: [99418112, 99422208) 99422208, 4096 00020000:00200000:0.0:1775764896.255372:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.255384:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24272, 24272] 00000008:00010000:0.0:1775764896.255389:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.255391:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.255399:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255408:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99418112 99422207) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255416:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255423:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255429:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.255437:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99418112, 99422208) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.255442:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24272 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.255459:0:18302:0:(rw26.c:858:ll_write_end()) pos 99418112, len 4096, copied 4096 00000080:00200000:0.0:1775764896.255463:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000024b9d162[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.255466:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006762121a 17ffffc0001001 3:0 ffff90c21282d8c0 24272 lru 00000080:00200000:0.0:1775764896.255472:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000018f52225 24272: 1< 0 - - > 2< 99418112 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1196032 - | + - + - > 5< + - + - | 0 - | 3792 - -> 00000080:00200000:0.0:1775764896.255474:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000024b9d162 00000080:00200000:0.0:1775764896.255475:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.255477:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.255485:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.255493:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.255499:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.255512:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.255523:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.255526:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.255529:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.255538:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.255541:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.255544:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.255548:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255555:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.255563:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.255566:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.255570:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.255573:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24272, 24272] 00000080:00200000:0.0:1775764896.255583:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.255587:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.255616:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99422208 i_blocks 90112 00000080:00200000:0.0:1775764896.255619:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.255635:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99422208, count: 4096 00000080:00200000:0.0:1775764896.255646:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99422208, bytes: 4096 00000080:00200000:0.0:1775764896.255649:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.255653:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24274, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.255654:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99422208, 99426304) 0 0 00000080:00200000:0.0:1775764896.255656:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.255662:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24273, 24273] 00020000:00200000:0.0:1775764896.255665:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99422208 chunk: [99422208, 99426304) 99426304, 4096 00020000:00200000:0.0:1775764896.255667:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.255679:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24273, 24273] 00000008:00010000:0.0:1775764896.255685:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.255687:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.255694:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255703:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99422208 99426303) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255713:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255720:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255727:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.255734:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99422208, 99426304) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.255739:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24273 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.255755:0:18302:0:(rw26.c:858:ll_write_end()) pos 99422208, len 4096, copied 4096 00000080:00200000:0.0:1775764896.255759:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000eb2fe500[3 00000000b99c1f5d 2 1 00000000897bd139] 00000080:00200000:0.0:1775764896.255762:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000032325d9e 17ffffc0001001 3:0 ffff90c21282db60 24273 lru 00000080:00200000:0.0:1775764896.255768:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000aab4a929 24273: 1< 0 - - > 2< 99422208 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1191936 - | + - + - > 5< + - + - | 0 - | 3793 - -> 00000080:00200000:0.0:1775764896.255770:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000eb2fe500 00000080:00200000:0.0:1775764896.255771:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.255773:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.255781:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.255789:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ada6f9aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.255796:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.255803:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.255814:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.255816:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.255818:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.255828:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.255831:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.255833:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.255837:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.255846:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.255853:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1f38 (ffff90c210c81bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.255856:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.255860:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.255863:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24273, 24273] 00000080:00200000:0.0:1775764896.255958:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.255962:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.256089:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99426304 i_blocks 90112 00000080:00200000:1.0:1775764896.256094:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.256125:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99426304, count: 4096 00000080:00200000:1.0:1775764896.256142:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99426304, bytes: 4096 00000080:00200000:1.0:1775764896.256145:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.256148:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24275, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.256150:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99426304, 99430400) 0 0 00000080:00200000:1.0:1775764896.256152:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.256157:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24274, 24274] 00020000:00200000:1.0:1775764896.256160:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99426304 chunk: [99426304, 99430400) 99430400, 4096 00020000:00200000:1.0:1775764896.256162:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.256176:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24274, 24274] 00000008:00010000:1.0:1775764896.256181:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.256184:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.256192:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256199:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99426304 99430399) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256207:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256213:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256219:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.256227:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99426304, 99430400) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.256231:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24274 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.256250:0:18302:0:(rw26.c:858:ll_write_end()) pos 99426304, len 4096, copied 4096 00000080:00200000:1.0:1775764896.256255:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000074007d6a[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.256257:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d13527b1 17ffffc0001001 3:0 ffff90c212828e00 24274 lru 00000080:00200000:1.0:1775764896.256262:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005c0b71d3 24274: 1< 0 - - > 2< 99426304 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1187840 - | + - + - > 5< + - + - | 0 - | 3794 - -> 00000080:00200000:1.0:1775764896.256264:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000074007d6a 00000080:00200000:1.0:1775764896.256265:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.256267:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.256275:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.256282:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.256289:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.256296:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.256307:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.256309:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.256312:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.256325:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.256327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.256330:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.256334:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256342:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.256349:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.256351:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.256355:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.256358:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24274, 24274] 00000080:00200000:1.0:1775764896.256375:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.256379:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.256409:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99430400 i_blocks 90112 00000080:00200000:1.0:1775764896.256413:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.256430:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99430400, count: 4096 00000080:00200000:1.0:1775764896.256439:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99430400, bytes: 4096 00000080:00200000:1.0:1775764896.256441:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.256444:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24276, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.256446:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99430400, 99434496) 0 0 00000080:00200000:1.0:1775764896.256448:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.256453:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24275, 24275] 00020000:00200000:1.0:1775764896.256456:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99430400 chunk: [99430400, 99434496) 99434496, 4096 00020000:00200000:1.0:1775764896.256458:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.256469:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24275, 24275] 00000008:00010000:1.0:1775764896.256474:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.256476:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.256484:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256493:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99430400 99434495) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256502:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256509:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256515:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.256523:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99430400, 99434496) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.256528:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24275 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.256545:0:18302:0:(rw26.c:858:ll_write_end()) pos 99430400, len 4096, copied 4096 00000080:00200000:1.0:1775764896.256550:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000008a0d23d[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.256552:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002b9247d5 17ffffc0001001 3:0 ffff90c212828540 24275 lru 00000080:00200000:1.0:1775764896.256557:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005bfef6cd 24275: 1< 0 - - > 2< 99430400 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1183744 - | + - + - > 5< + - + - | 0 - | 3795 - -> 00000080:00200000:1.0:1775764896.256559:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000008a0d23d 00000080:00200000:1.0:1775764896.256560:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.256561:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.256569:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.256576:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.256582:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.256594:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.256605:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.256608:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.256610:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.256624:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.256627:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.256629:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.256633:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256641:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.256649:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.256652:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.256656:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.256659:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24275, 24275] 00000080:00200000:1.0:1775764896.256670:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.256673:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.256704:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99434496 i_blocks 90112 00000080:00200000:1.0:1775764896.256707:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.256722:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99434496, count: 4096 00000080:00200000:1.0:1775764896.256733:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99434496, bytes: 4096 00000080:00200000:1.0:1775764896.256735:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.256739:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24277, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.256740:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99434496, 99438592) 0 0 00000080:00200000:1.0:1775764896.256743:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.256747:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24276, 24276] 00020000:00200000:1.0:1775764896.256750:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99434496 chunk: [99434496, 99438592) 99438592, 4096 00020000:00200000:1.0:1775764896.256752:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.256763:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24276, 24276] 00000008:00010000:1.0:1775764896.256768:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.256771:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.256779:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256787:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99434496 99438591) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256795:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256802:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256809:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.256817:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99434496, 99438592) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.256823:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24276 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.256839:0:18302:0:(rw26.c:858:ll_write_end()) pos 99434496, len 4096, copied 4096 00000080:00200000:1.0:1775764896.256843:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a11b671d[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.256846:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000013013f55 17ffffc0001001 3:0 ffff90c212828460 24276 lru 00000080:00200000:1.0:1775764896.256851:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007b1bd599 24276: 1< 0 - - > 2< 99434496 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1179648 - | + - + - > 5< + - + - | 0 - | 3796 - -> 00000080:00200000:1.0:1775764896.256854:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a11b671d 00000080:00200000:1.0:1775764896.256855:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.256857:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.256866:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.256895:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.256902:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.256910:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.256921:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.256925:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.256927:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.256937:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.256939:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.256942:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.256945:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.256954:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.256961:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.256964:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.256969:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.256972:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24276, 24276] 00000080:00200000:1.0:1775764896.256983:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.256986:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.257018:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99438592 i_blocks 90112 00000080:00200000:1.0:1775764896.257022:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.257038:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99438592, count: 4096 00000080:00200000:1.0:1775764896.257048:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99438592, bytes: 4096 00000080:00200000:1.0:1775764896.257051:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.257054:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24278, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.257056:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99438592, 99442688) 0 0 00000080:00200000:1.0:1775764896.257058:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.257063:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24277, 24277] 00020000:00200000:1.0:1775764896.257067:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99438592 chunk: [99438592, 99442688) 99442688, 4096 00020000:00200000:1.0:1775764896.257069:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.257080:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24277, 24277] 00000008:00010000:1.0:1775764896.257086:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.257088:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.257096:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257105:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99438592 99442687) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257126:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257135:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257147:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.257155:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99438592, 99442688) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.257160:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24277 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.257176:0:18302:0:(rw26.c:858:ll_write_end()) pos 99438592, len 4096, copied 4096 00000080:00200000:1.0:1775764896.257180:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004cce59d4[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.257183:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000abcdf28d 17ffffc0001001 3:0 ffff90c212828a80 24277 lru 00000080:00200000:1.0:1775764896.257189:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005233d581 24277: 1< 0 - - > 2< 99438592 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1175552 - | + - + - > 5< + - + - | 0 - | 3797 - -> 00000080:00200000:1.0:1775764896.257191:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004cce59d4 00000080:00200000:1.0:1775764896.257192:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.257194:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.257202:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.257210:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.257217:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.257224:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.257236:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.257238:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.257241:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.257251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.257253:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.257256:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.257260:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257269:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.257278:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.257280:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.257285:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.257288:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24277, 24277] 00000080:00200000:1.0:1775764896.257299:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.257302:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.257332:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99442688 i_blocks 90112 00000080:00200000:1.0:1775764896.257336:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.257353:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99442688, count: 4096 00000080:00200000:1.0:1775764896.257364:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99442688, bytes: 4096 00000080:00200000:1.0:1775764896.257367:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.257372:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24279, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.257374:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99442688, 99446784) 0 0 00000080:00200000:1.0:1775764896.257376:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.257382:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24278, 24278] 00020000:00200000:1.0:1775764896.257385:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99442688 chunk: [99442688, 99446784) 99446784, 4096 00020000:00200000:1.0:1775764896.257387:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.257399:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24278, 24278] 00000008:00010000:1.0:1775764896.257404:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.257406:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.257414:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257423:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99442688 99446783) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257432:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257439:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257446:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.257454:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99442688, 99446784) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.257460:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24278 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.257476:0:18302:0:(rw26.c:858:ll_write_end()) pos 99442688, len 4096, copied 4096 00000080:00200000:1.0:1775764896.257481:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006e2ee4be[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.257483:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f2a94c2f 17ffffc0001001 3:0 ffff90c212828700 24278 lru 00000080:00200000:1.0:1775764896.257489:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000056f674c9 24278: 1< 0 - - > 2< 99442688 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1171456 - | + - + - > 5< + - + - | 0 - | 3798 - -> 00000080:00200000:1.0:1775764896.257491:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006e2ee4be 00000080:00200000:1.0:1775764896.257492:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.257494:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.257502:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.257509:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.257516:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.257523:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.257534:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.257536:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.257539:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.257549:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.257551:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.257554:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.257558:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257565:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.257574:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.257577:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.257580:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.257583:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24278, 24278] 00000080:00200000:1.0:1775764896.257594:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.257597:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.257627:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99446784 i_blocks 90112 00000080:00200000:1.0:1775764896.257631:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.257647:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99446784, count: 4096 00000080:00200000:1.0:1775764896.257657:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99446784, bytes: 4096 00000080:00200000:1.0:1775764896.257660:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.257664:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24280, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.257665:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99446784, 99450880) 0 0 00000080:00200000:1.0:1775764896.257668:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.257672:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24279, 24279] 00020000:00200000:1.0:1775764896.257676:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99446784 chunk: [99446784, 99450880) 99450880, 4096 00020000:00200000:1.0:1775764896.257678:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.257690:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24279, 24279] 00000008:00010000:1.0:1775764896.257695:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.257697:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.257705:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257714:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99446784 99450879) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257722:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257730:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257737:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.257744:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99446784, 99450880) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.257750:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24279 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.257767:0:18302:0:(rw26.c:858:ll_write_end()) pos 99446784, len 4096, copied 4096 00000080:00200000:1.0:1775764896.257771:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e291c757[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.257773:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000548ec23c 17ffffc0001001 3:0 ffff90c212828c40 24279 lru 00000080:00200000:1.0:1775764896.257780:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ef3f6511 24279: 1< 0 - - > 2< 99446784 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1167360 - | + - + - > 5< + - + - | 0 - | 3799 - -> 00000080:00200000:1.0:1775764896.257782:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e291c757 00000080:00200000:1.0:1775764896.257783:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.257785:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.257792:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.257800:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.257807:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.257814:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.257824:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.257827:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.257829:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.257839:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.257842:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.257844:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.257848:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.257855:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.257864:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.257866:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.257966:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.257970:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24279, 24279] 00000080:00200000:1.0:1775764896.257981:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.257985:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.258015:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99450880 i_blocks 90112 00000080:00200000:1.0:1775764896.258019:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.258035:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99450880, count: 4096 00000080:00200000:1.0:1775764896.258047:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99450880, bytes: 4096 00000080:00200000:1.0:1775764896.258050:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.258054:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24281, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.258055:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99450880, 99454976) 0 0 00000080:00200000:1.0:1775764896.258058:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.258062:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24280, 24280] 00020000:00200000:1.0:1775764896.258065:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99450880 chunk: [99450880, 99454976) 99454976, 4096 00020000:00200000:1.0:1775764896.258068:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.258079:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24280, 24280] 00000008:00010000:1.0:1775764896.258084:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.258087:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.258096:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258105:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99450880 99454975) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258125:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258138:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258144:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.258152:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99450880, 99454976) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.258158:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24280 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.258173:0:18302:0:(rw26.c:858:ll_write_end()) pos 99450880, len 4096, copied 4096 00000080:00200000:1.0:1775764896.258177:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e99f5038[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.258179:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000017691a2d 17ffffc0001001 3:0 ffff90c2128288c0 24280 lru 00000080:00200000:1.0:1775764896.258185:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002d0e163a 24280: 1< 0 - - > 2< 99450880 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1163264 - | + - + - > 5< + - + - | 0 - | 3800 - -> 00000080:00200000:1.0:1775764896.258187:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e99f5038 00000080:00200000:1.0:1775764896.258188:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.258190:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.258197:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.258205:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.258212:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.258219:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.258236:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.258239:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.258241:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.258251:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.258254:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.258256:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.258260:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258268:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.258276:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.258278:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.258282:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.258285:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24280, 24280] 00000080:00200000:1.0:1775764896.258295:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.258298:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.258328:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99454976 i_blocks 90112 00000080:00200000:1.0:1775764896.258332:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.258347:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99454976, count: 4096 00000080:00200000:1.0:1775764896.258358:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99454976, bytes: 4096 00000080:00200000:1.0:1775764896.258360:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.258364:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24282, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.258365:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99454976, 99459072) 0 0 00000080:00200000:1.0:1775764896.258368:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.258373:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24281, 24281] 00020000:00200000:1.0:1775764896.258376:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99454976 chunk: [99454976, 99459072) 99459072, 4096 00020000:00200000:1.0:1775764896.258378:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.258390:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24281, 24281] 00000008:00010000:1.0:1775764896.258394:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.258397:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.258405:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258413:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99454976 99459071) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258421:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258428:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258435:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.258443:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99454976, 99459072) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.258448:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24281 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.258465:0:18302:0:(rw26.c:858:ll_write_end()) pos 99454976, len 4096, copied 4096 00000080:00200000:1.0:1775764896.258469:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b810d78f[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.258472:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c45688ca 17ffffc0001001 3:0 ffff90c212828b60 24281 lru 00000080:00200000:1.0:1775764896.258477:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d2f144c7 24281: 1< 0 - - > 2< 99454976 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1159168 - | + - + - > 5< + - + - | 0 - | 3801 - -> 00000080:00200000:1.0:1775764896.258479:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b810d78f 00000080:00200000:1.0:1775764896.258480:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.258482:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.258491:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.258498:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.258504:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.258511:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.258522:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.258525:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.258527:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.258537:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.258540:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.258542:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.258546:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258554:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.258562:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.258564:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.258567:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.258571:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24281, 24281] 00000080:00200000:1.0:1775764896.258582:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.258584:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.258613:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99459072 i_blocks 90112 00000080:00200000:1.0:1775764896.258616:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.258632:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99459072, count: 4096 00000080:00200000:1.0:1775764896.258643:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99459072, bytes: 4096 00000080:00200000:1.0:1775764896.258645:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.258649:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24283, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.258650:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99459072, 99463168) 0 0 00000080:00200000:1.0:1775764896.258652:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.258657:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24282, 24282] 00020000:00200000:1.0:1775764896.258660:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99459072 chunk: [99459072, 99463168) 99463168, 4096 00020000:00200000:1.0:1775764896.258662:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.258673:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24282, 24282] 00000008:00010000:1.0:1775764896.258678:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.258680:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.258687:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258696:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99459072 99463167) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258704:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258712:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258719:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.258727:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99459072, 99463168) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.258731:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24282 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.258748:0:18302:0:(rw26.c:858:ll_write_end()) pos 99459072, len 4096, copied 4096 00000080:00200000:1.0:1775764896.258752:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a2a8ee5f[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.258755:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005d172973 17ffffc0001001 3:0 ffff90c212828000 24282 lru 00000080:00200000:1.0:1775764896.258760:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000587a83b3 24282: 1< 0 - - > 2< 99459072 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1155072 - | + - + - > 5< + - + - | 0 - | 3802 - -> 00000080:00200000:1.0:1775764896.258762:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a2a8ee5f 00000080:00200000:1.0:1775764896.258763:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.258765:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.258772:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.258780:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.258787:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.258794:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.258805:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.258808:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.258810:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.258818:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.258821:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.258823:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.258827:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.258834:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.258841:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.258843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.258847:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.258849:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24282, 24282] 00000080:00200000:1.0:1775764896.258861:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.258864:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.258973:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99463168 i_blocks 90112 00000080:00200000:1.0:1775764896.258976:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.258993:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99463168, count: 4096 00000080:00200000:1.0:1775764896.259003:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99463168, bytes: 4096 00000080:00200000:1.0:1775764896.259006:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.259009:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24284, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.259011:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99463168, 99467264) 0 0 00000080:00200000:1.0:1775764896.259013:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.259018:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24283, 24283] 00020000:00200000:1.0:1775764896.259021:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99463168 chunk: [99463168, 99467264) 99467264, 4096 00020000:00200000:1.0:1775764896.259023:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.259033:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24283, 24283] 00000008:00010000:1.0:1775764896.259039:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.259041:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.259048:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259057:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99463168 99467263) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259067:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259075:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259081:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.259089:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99463168, 99467264) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.259094:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24283 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.259132:0:18302:0:(rw26.c:858:ll_write_end()) pos 99463168, len 4096, copied 4096 00000080:00200000:1.0:1775764896.259137:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b644fc09[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.259139:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000891ed4db 17ffffc0001001 3:0 ffff90c212829a80 24283 lru 00000080:00200000:1.0:1775764896.259144:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000594e52c 24283: 1< 0 - - > 2< 99463168 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1150976 - | + - + - > 5< + - + - | 0 - | 3803 - -> 00000080:00200000:1.0:1775764896.259146:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b644fc09 00000080:00200000:1.0:1775764896.259147:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.259149:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.259157:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.259165:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.259172:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.259184:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.259196:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.259199:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.259201:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.259211:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.259213:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.259216:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.259220:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259227:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.259234:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.259236:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.259240:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.259243:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24283, 24283] 00000080:00200000:1.0:1775764896.259253:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.259256:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.259286:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99467264 i_blocks 90112 00000080:00200000:1.0:1775764896.259289:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.259304:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99467264, count: 4096 00000080:00200000:1.0:1775764896.259313:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99467264, bytes: 4096 00000080:00200000:1.0:1775764896.259315:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.259319:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24285, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.259320:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99467264, 99471360) 0 0 00000080:00200000:1.0:1775764896.259322:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.259327:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24284, 24284] 00020000:00200000:1.0:1775764896.259330:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99467264 chunk: [99467264, 99471360) 99471360, 4096 00020000:00200000:1.0:1775764896.259333:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.259343:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24284, 24284] 00000008:00010000:1.0:1775764896.259348:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.259350:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.259358:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259367:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99467264 99471359) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259374:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259382:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259389:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.259398:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99467264, 99471360) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.259403:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24284 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.259420:0:18302:0:(rw26.c:858:ll_write_end()) pos 99467264, len 4096, copied 4096 00000080:00200000:1.0:1775764896.259424:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d468de80[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.259427:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000083711ee7 17ffffc0001001 3:0 ffff90c212829700 24284 lru 00000080:00200000:1.0:1775764896.259431:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004dd04fce 24284: 1< 0 - - > 2< 99467264 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1146880 - | + - + - > 5< + - + - | 0 - | 3804 - -> 00000080:00200000:1.0:1775764896.259434:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d468de80 00000080:00200000:1.0:1775764896.259435:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.259437:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.259444:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.259452:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.259458:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.259465:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.259476:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.259479:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.259481:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.259490:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.259493:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.259495:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.259499:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259507:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.259515:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.259518:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.259521:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.259524:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24284, 24284] 00000080:00200000:1.0:1775764896.259535:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.259538:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.259568:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99471360 i_blocks 90112 00000080:00200000:1.0:1775764896.259572:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.259587:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99471360, count: 4096 00000080:00200000:1.0:1775764896.259598:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99471360, bytes: 4096 00000080:00200000:1.0:1775764896.259601:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.259604:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24286, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.259606:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99471360, 99475456) 0 0 00000080:00200000:1.0:1775764896.259608:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.259613:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24285, 24285] 00020000:00200000:1.0:1775764896.259616:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99471360 chunk: [99471360, 99475456) 99475456, 4096 00020000:00200000:1.0:1775764896.259618:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.259629:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24285, 24285] 00000008:00010000:1.0:1775764896.259635:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.259637:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.259644:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259653:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99471360 99475455) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259660:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259668:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259680:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.259688:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99471360, 99475456) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.259693:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24285 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.259716:0:18302:0:(rw26.c:858:ll_write_end()) pos 99471360, len 4096, copied 4096 00000080:00200000:1.0:1775764896.259721:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e8b7997f[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.259723:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000033be21f 17ffffc0001021 2:0 ffff90c212829c40 24285 lru 00000080:00200000:1.0:1775764896.259729:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000392aedb 24285: 1< 0 - - > 2< 99471360 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1142784 - | + - + - > 5< + - + - | 0 - | 3805 - -> 00000080:00200000:1.0:1775764896.259731:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e8b7997f 00000080:00200000:1.0:1775764896.259732:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.259734:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.259742:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.259750:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.259756:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.259763:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.259774:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.259777:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.259780:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.259790:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.259792:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.259795:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.259799:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.259807:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.259815:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.259818:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.259822:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.259824:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24285, 24285] 00000080:00200000:1.0:1775764896.259836:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.259839:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.259868:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99475456 i_blocks 90112 00000080:00200000:1.0:1775764896.259944:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.259962:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99475456, count: 4096 00000080:00200000:1.0:1775764896.259973:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99475456, bytes: 4096 00000080:00200000:1.0:1775764896.259976:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.259980:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24287, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.259981:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99475456, 99479552) 0 0 00000080:00200000:1.0:1775764896.259984:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.259989:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24286, 24286] 00020000:00200000:1.0:1775764896.259992:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99475456 chunk: [99475456, 99479552) 99479552, 4096 00020000:00200000:1.0:1775764896.259995:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.260006:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24286, 24286] 00000008:00010000:1.0:1775764896.260012:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.260014:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.260022:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260031:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99475456 99479551) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260038:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260045:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260051:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.260058:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99475456, 99479552) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.260063:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24286 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.260079:0:18302:0:(rw26.c:858:ll_write_end()) pos 99475456, len 4096, copied 4096 00000080:00200000:1.0:1775764896.260084:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cbdaf873[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.260086:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003e26dc85 17ffffc0001001 3:0 ffff90c2128298c0 24286 lru 00000080:00200000:1.0:1775764896.260091:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000762df591 24286: 1< 0 - - > 2< 99475456 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1138688 - | + - + - > 5< + - + - | 0 - | 3806 - -> 00000080:00200000:1.0:1775764896.260093:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cbdaf873 00000080:00200000:1.0:1775764896.260094:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.260096:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.260103:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260111:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260128:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260133:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.260141:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.260143:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.260145:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.260152:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.260153:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.260155:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.260158:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260163:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.260168:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.260169:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.260171:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.260173:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24286, 24286] 00000080:00200000:1.0:1775764896.260181:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.260183:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.260202:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99479552 i_blocks 90112 00000080:00200000:1.0:1775764896.260204:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.260215:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99479552, count: 4096 00000080:00200000:1.0:1775764896.260221:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99479552, bytes: 4096 00000080:00200000:1.0:1775764896.260223:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.260226:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24288, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.260227:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99479552, 99483648) 0 0 00000080:00200000:1.0:1775764896.260228:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.260232:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24287, 24287] 00020000:00200000:1.0:1775764896.260234:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99479552 chunk: [99479552, 99483648) 99483648, 4096 00020000:00200000:1.0:1775764896.260235:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.260242:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24287, 24287] 00000008:00010000:1.0:1775764896.260245:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.260246:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.260252:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260258:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99479552 99483647) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260263:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260268:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260272:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.260276:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99479552, 99483648) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.260279:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24287 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.260295:0:18302:0:(rw26.c:858:ll_write_end()) pos 99479552, len 4096, copied 4096 00000080:00200000:1.0:1775764896.260297:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001795c4ea[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.260299:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000046b79ff4 17ffffc0001001 3:0 ffff90c212829b60 24287 lru 00000080:00200000:1.0:1775764896.260303:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ba058523 24287: 1< 0 - - > 2< 99479552 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1134592 - | + - + - > 5< + - + - | 0 - | 3807 - -> 00000080:00200000:1.0:1775764896.260304:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001795c4ea 00000080:00200000:1.0:1775764896.260305:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.260306:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.260311:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260316:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260321:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260325:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.260334:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.260337:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.260339:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.260347:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.260349:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.260351:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.260355:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260362:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.260370:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.260372:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.260376:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.260379:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24287, 24287] 00000080:00200000:1.0:1775764896.260390:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.260393:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.260419:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99483648 i_blocks 90112 00000080:00200000:1.0:1775764896.260421:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.260432:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99483648, count: 4096 00000080:00200000:1.0:1775764896.260438:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99483648, bytes: 4096 00000080:00200000:1.0:1775764896.260440:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.260443:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24289, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.260444:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99483648, 99487744) 0 0 00000080:00200000:1.0:1775764896.260445:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.260449:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24288, 24288] 00020000:00200000:1.0:1775764896.260451:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99483648 chunk: [99483648, 99487744) 99487744, 4096 00020000:00200000:1.0:1775764896.260452:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.260459:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24288, 24288] 00000008:00010000:1.0:1775764896.260462:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.260464:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.260469:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260475:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99483648 99487743) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260481:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260493:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260499:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.260507:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99483648, 99487744) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.260512:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24288 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.260528:0:18302:0:(rw26.c:858:ll_write_end()) pos 99483648, len 4096, copied 4096 00000080:00200000:1.0:1775764896.260532:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000473a00bb[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.260535:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000029a7bda1 17ffffc0001001 3:0 ffff90c212829000 24288 lru 00000080:00200000:1.0:1775764896.260539:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000eadcbf16 24288: 1< 0 - - > 2< 99483648 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1130496 - | + - + - > 5< + - + - | 0 - | 3808 - -> 00000080:00200000:1.0:1775764896.260541:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000473a00bb 00000080:00200000:1.0:1775764896.260543:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.260545:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.260552:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260560:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260567:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260575:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.260591:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.260595:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.260597:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.260606:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.260609:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.260611:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.260615:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260623:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.260631:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.260633:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.260637:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.260640:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24288, 24288] 00000080:00200000:1.0:1775764896.260665:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.260668:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.260697:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99487744 i_blocks 90112 00000080:00200000:1.0:1775764896.260700:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.260716:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99487744, count: 4096 00000080:00200000:1.0:1775764896.260727:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99487744, bytes: 4096 00000080:00200000:1.0:1775764896.260729:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.260733:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24290, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.260734:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99487744, 99491840) 0 0 00000080:00200000:1.0:1775764896.260737:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.260742:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24289, 24289] 00020000:00200000:1.0:1775764896.260746:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99487744 chunk: [99487744, 99491840) 99491840, 4096 00020000:00200000:1.0:1775764896.260748:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.260759:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24289, 24289] 00000008:00010000:1.0:1775764896.260764:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.260766:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.260774:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260783:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99487744 99491839) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260792:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260799:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260806:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.260813:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99487744, 99491840) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.260818:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24289 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.260833:0:18302:0:(rw26.c:858:ll_write_end()) pos 99487744, len 4096, copied 4096 00000080:00200000:1.0:1775764896.260838:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003197ba6a[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.260841:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000016eb17bf 17ffffc0001001 3:0 ffff90c212829ee0 24289 lru 00000080:00200000:1.0:1775764896.260846:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000066fce246 24289: 1< 0 - - > 2< 99487744 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1126400 - | + - + - > 5< + - + - | 0 - | 3809 - -> 00000080:00200000:1.0:1775764896.260848:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003197ba6a 00000080:00200000:1.0:1775764896.260849:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.260851:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.260859:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260867:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260895:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.260902:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.260914:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.260917:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.260919:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.260928:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.260931:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.260933:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.260937:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.260945:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.260954:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.260956:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.260960:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.260963:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24289, 24289] 00000080:00200000:1.0:1775764896.260974:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.260977:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.261006:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99491840 i_blocks 90112 00000080:00200000:1.0:1775764896.261009:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.261025:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99491840, count: 4096 00000080:00200000:1.0:1775764896.261035:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99491840, bytes: 4096 00000080:00200000:1.0:1775764896.261037:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.261041:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24291, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.261042:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99491840, 99495936) 0 0 00000080:00200000:1.0:1775764896.261044:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.261049:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24290, 24290] 00020000:00200000:1.0:1775764896.261052:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99491840 chunk: [99491840, 99495936) 99495936, 4096 00020000:00200000:1.0:1775764896.261055:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.261066:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24290, 24290] 00000008:00010000:1.0:1775764896.261070:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.261073:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.261080:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261089:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99491840 99495935) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261097:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261105:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261111:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.261133:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99491840, 99495936) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.261138:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24290 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.261154:0:18302:0:(rw26.c:858:ll_write_end()) pos 99491840, len 4096, copied 4096 00000080:00200000:1.0:1775764896.261159:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005b9319eb[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.261161:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c0200e56 17ffffc0001001 3:0 ffff90c2128290e0 24290 lru 00000080:00200000:1.0:1775764896.261164:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000416655ea 24290: 1< 0 - - > 2< 99491840 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1122304 - | + - + - > 5< + - + - | 0 - | 3810 - -> 00000080:00200000:1.0:1775764896.261166:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005b9319eb 00000080:00200000:1.0:1775764896.261166:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.261168:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.261174:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.261178:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.261182:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.261187:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.261194:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.261195:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.261196:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.261203:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.261205:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.261206:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.261209:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261214:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.261220:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.261221:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.261223:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.261225:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24290, 24290] 00000080:00200000:1.0:1775764896.261233:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.261235:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.261254:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99495936 i_blocks 90112 00000080:00200000:1.0:1775764896.261256:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.261267:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99495936, count: 4096 00000080:00200000:1.0:1775764896.261273:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99495936, bytes: 4096 00000080:00200000:1.0:1775764896.261274:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.261277:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24292, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.261278:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99495936, 99500032) 0 0 00000080:00200000:1.0:1775764896.261279:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.261283:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24291, 24291] 00020000:00200000:1.0:1775764896.261285:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99495936 chunk: [99495936, 99500032) 99500032, 4096 00020000:00200000:1.0:1775764896.261286:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.261293:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24291, 24291] 00000008:00010000:1.0:1775764896.261296:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.261298:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.261303:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261309:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99495936 99500031) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261315:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261319:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261323:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.261328:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99495936, 99500032) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.261333:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24291 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.261348:0:18302:0:(rw26.c:858:ll_write_end()) pos 99495936, len 4096, copied 4096 00000080:00200000:1.0:1775764896.261352:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009217cd70[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.261354:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001c9329bf 17ffffc0001001 3:0 ffff90c2128291c0 24291 lru 00000080:00200000:1.0:1775764896.261359:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007398c382 24291: 1< 0 - - > 2< 99495936 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1118208 - | + - + - > 5< + - + - | 0 - | 3811 - -> 00000080:00200000:1.0:1775764896.261361:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009217cd70 00000080:00200000:1.0:1775764896.261362:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.261364:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.261371:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.261380:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.261387:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.261400:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.261411:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.261413:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.261414:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.261421:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.261423:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.261425:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.261427:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261432:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.261439:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.261441:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.261445:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.261447:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24291, 24291] 00000080:00200000:1.0:1775764896.261458:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.261461:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.261490:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99500032 i_blocks 90112 00000080:00200000:1.0:1775764896.261493:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.261509:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99500032, count: 4096 00000080:00200000:1.0:1775764896.261520:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99500032, bytes: 4096 00000080:00200000:1.0:1775764896.261523:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.261527:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24293, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.261528:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99500032, 99504128) 0 0 00000080:00200000:1.0:1775764896.261530:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.261535:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24292, 24292] 00020000:00200000:1.0:1775764896.261538:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99500032 chunk: [99500032, 99504128) 99504128, 4096 00020000:00200000:1.0:1775764896.261540:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.261552:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24292, 24292] 00000008:00010000:1.0:1775764896.261557:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.261559:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.261566:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261575:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99500032 99504127) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261584:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261591:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261598:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.261606:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99500032, 99504128) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.261611:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24292 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.261626:0:18302:0:(rw26.c:858:ll_write_end()) pos 99500032, len 4096, copied 4096 00000080:00200000:1.0:1775764896.261630:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005acb6642[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.261633:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007f21f070 17ffffc0001001 3:0 ffff90c2128297e0 24292 lru 00000080:00200000:1.0:1775764896.261639:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a3554624 24292: 1< 0 - - > 2< 99500032 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1114112 - | + - + - > 5< + - + - | 0 - | 3812 - -> 00000080:00200000:1.0:1775764896.261641:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005acb6642 00000080:00200000:1.0:1775764896.261642:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.261644:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.261652:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.261660:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.261668:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.261675:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.261687:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.261689:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.261692:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.261701:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.261703:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.261706:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.261710:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261718:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.261726:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.261729:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.261733:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.261736:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24292, 24292] 00000080:00200000:1.0:1775764896.261747:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.261751:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.261781:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99504128 i_blocks 90112 00000080:00200000:1.0:1775764896.261784:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.261799:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99504128, count: 4096 00000080:00200000:1.0:1775764896.261810:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99504128, bytes: 4096 00000080:00200000:1.0:1775764896.261812:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.261816:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24294, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.261818:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99504128, 99508224) 0 0 00000080:00200000:1.0:1775764896.261820:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.261825:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24293, 24293] 00020000:00200000:1.0:1775764896.261828:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99504128 chunk: [99504128, 99508224) 99508224, 4096 00020000:00200000:1.0:1775764896.261831:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.261841:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24293, 24293] 00000008:00010000:1.0:1775764896.261847:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.261849:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.261857:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261865:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99504128 99508223) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261958:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261966:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.261977:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.261985:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99504128, 99508224) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.261990:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24293 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.262008:0:18302:0:(rw26.c:858:ll_write_end()) pos 99504128, len 4096, copied 4096 00000080:00200000:1.0:1775764896.262012:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ed7c6f03[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.262014:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007579b9ea 17ffffc0001001 3:0 ffff90c2128299a0 24293 lru 00000080:00200000:1.0:1775764896.262020:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a4640034 24293: 1< 0 - - > 2< 99504128 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1110016 - | + - + - > 5< + - + - | 0 - | 3813 - -> 00000080:00200000:1.0:1775764896.262021:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ed7c6f03 00000080:00200000:1.0:1775764896.262022:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.262024:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.262032:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262039:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262046:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262053:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.262065:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.262068:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.262070:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.262080:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.262083:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.262085:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.262089:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262096:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.262105:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.262108:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.262122:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.262125:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24293, 24293] 00000080:00200000:1.0:1775764896.262136:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.262139:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.262169:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99508224 i_blocks 90112 00000080:00200000:1.0:1775764896.262173:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.262186:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99508224, count: 4096 00000080:00200000:1.0:1775764896.262193:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99508224, bytes: 4096 00000080:00200000:1.0:1775764896.262195:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.262197:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24295, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.262198:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99508224, 99512320) 0 0 00000080:00200000:1.0:1775764896.262200:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.262204:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24294, 24294] 00020000:00200000:1.0:1775764896.262206:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99508224 chunk: [99508224, 99512320) 99512320, 4096 00020000:00200000:1.0:1775764896.262207:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.262214:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24294, 24294] 00000008:00010000:1.0:1775764896.262217:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.262219:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.262225:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262231:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99508224 99512319) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262237:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262241:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262245:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.262249:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99508224, 99512320) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.262252:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24294 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.262265:0:18302:0:(rw26.c:858:ll_write_end()) pos 99508224, len 4096, copied 4096 00000080:00200000:1.0:1775764896.262268:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003131b2be[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.262269:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000361a43ae 17ffffc0001001 3:0 ffff90c2128292a0 24294 lru 00000080:00200000:1.0:1775764896.262273:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000baba0201 24294: 1< 0 - - > 2< 99508224 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1105920 - | + - + - > 5< + - + - | 0 - | 3814 - -> 00000080:00200000:1.0:1775764896.262274:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003131b2be 00000080:00200000:1.0:1775764896.262275:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.262277:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.262281:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262286:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262291:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262296:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.262302:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.262304:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.262305:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.262312:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.262313:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.262314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.262317:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262322:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.262327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.262329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.262333:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.262335:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24294, 24294] 00000080:00200000:1.0:1775764896.262345:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.262348:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.262375:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99512320 i_blocks 90112 00000080:00200000:1.0:1775764896.262379:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.262394:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99512320, count: 4096 00000080:00200000:1.0:1775764896.262405:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99512320, bytes: 4096 00000080:00200000:1.0:1775764896.262407:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.262410:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24296, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.262411:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99512320, 99516416) 0 0 00000080:00200000:1.0:1775764896.262413:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.262416:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24295, 24295] 00020000:00200000:1.0:1775764896.262419:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99512320 chunk: [99512320, 99516416) 99516416, 4096 00020000:00200000:1.0:1775764896.262420:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.262427:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24295, 24295] 00000008:00010000:1.0:1775764896.262430:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.262432:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.262438:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262444:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99512320 99516415) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262452:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262459:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262465:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.262472:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99512320, 99516416) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.262477:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24295 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.262493:0:18302:0:(rw26.c:858:ll_write_end()) pos 99512320, len 4096, copied 4096 00000080:00200000:1.0:1775764896.262497:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ea74080c[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.262500:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005c13466d 17ffffc0001001 3:0 ffff90c212829d20 24295 lru 00000080:00200000:1.0:1775764896.262506:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000026782de0 24295: 1< 0 - - > 2< 99512320 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1101824 - | + - + - > 5< + - + - | 0 - | 3815 - -> 00000080:00200000:1.0:1775764896.262508:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ea74080c 00000080:00200000:1.0:1775764896.262509:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.262511:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.262519:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262526:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262533:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262541:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.262552:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.262555:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.262557:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.262566:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.262569:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.262571:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.262575:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262583:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.262591:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.262594:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.262598:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.262601:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24295, 24295] 00000080:00200000:1.0:1775764896.262611:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.262614:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.262643:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99516416 i_blocks 90112 00000080:00200000:1.0:1775764896.262647:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.262663:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99516416, count: 4096 00000080:00200000:1.0:1775764896.262673:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99516416, bytes: 4096 00000080:00200000:1.0:1775764896.262676:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.262679:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24297, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.262681:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99516416, 99520512) 0 0 00000080:00200000:1.0:1775764896.262683:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.262688:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24296, 24296] 00020000:00200000:1.0:1775764896.262691:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99516416 chunk: [99516416, 99520512) 99520512, 4096 00020000:00200000:1.0:1775764896.262693:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.262704:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24296, 24296] 00000008:00010000:1.0:1775764896.262709:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.262712:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.262719:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262728:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99516416 99520511) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262737:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262749:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262756:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.262764:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99516416, 99520512) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.262769:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24296 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.262785:0:18302:0:(rw26.c:858:ll_write_end()) pos 99516416, len 4096, copied 4096 00000080:00200000:1.0:1775764896.262789:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000085f649ef[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.262792:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000278606ff 17ffffc0001001 3:0 ffff90c212829380 24296 lru 00000080:00200000:1.0:1775764896.262797:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006eaf396d 24296: 1< 0 - - > 2< 99516416 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1097728 - | + - + - > 5< + - + - | 0 - | 3816 - -> 00000080:00200000:1.0:1775764896.262799:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000085f649ef 00000080:00200000:1.0:1775764896.262800:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.262802:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.262809:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262817:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262824:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.262832:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.262849:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.262852:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.262854:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.262863:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.262866:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.262954:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.262960:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.262968:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.262977:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.262979:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.262984:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.262987:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24296, 24296] 00000080:00200000:1.0:1775764896.262998:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.263001:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.263023:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99520512 i_blocks 90112 00000080:00200000:1.0:1775764896.263026:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.263037:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99520512, count: 4096 00000080:00200000:1.0:1775764896.263043:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99520512, bytes: 4096 00000080:00200000:1.0:1775764896.263045:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.263047:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24298, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.263048:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99520512, 99524608) 0 0 00000080:00200000:1.0:1775764896.263050:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.263054:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24297, 24297] 00020000:00200000:1.0:1775764896.263056:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99520512 chunk: [99520512, 99524608) 99524608, 4096 00020000:00200000:1.0:1775764896.263058:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.263064:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24297, 24297] 00000008:00010000:1.0:1775764896.263068:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.263069:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.263075:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263081:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99520512 99524607) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263086:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263091:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263095:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.263100:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99520512, 99524608) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.263103:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24297 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.263128:0:18302:0:(rw26.c:858:ll_write_end()) pos 99520512, len 4096, copied 4096 00000080:00200000:1.0:1775764896.263132:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000044081552[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.263134:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005d30f0ee 17ffffc0001001 3:0 ffff90c212829620 24297 lru 00000080:00200000:1.0:1775764896.263137:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004c3c7af5 24297: 1< 0 - - > 2< 99520512 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1093632 - | + - + - > 5< + - + - | 0 - | 3817 - -> 00000080:00200000:1.0:1775764896.263139:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000044081552 00000080:00200000:1.0:1775764896.263139:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.263141:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.263147:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263152:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263156:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263160:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.263167:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.263169:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.263170:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.263177:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.263178:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.263180:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.263182:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263187:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.263192:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.263194:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.263196:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.263198:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24297, 24297] 00000080:00200000:1.0:1775764896.263205:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.263207:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.263226:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99524608 i_blocks 90112 00000080:00200000:1.0:1775764896.263229:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.263239:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99524608, count: 4096 00000080:00200000:1.0:1775764896.263245:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99524608, bytes: 4096 00000080:00200000:1.0:1775764896.263247:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.263249:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24299, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.263250:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99524608, 99528704) 0 0 00000080:00200000:1.0:1775764896.263252:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.263255:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24298, 24298] 00020000:00200000:1.0:1775764896.263257:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99524608 chunk: [99524608, 99528704) 99528704, 4096 00020000:00200000:1.0:1775764896.263259:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.263265:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24298, 24298] 00000008:00010000:1.0:1775764896.263269:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.263270:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.263275:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263281:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99524608 99528703) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263286:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263291:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263295:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.263300:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99524608, 99528704) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.263303:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24298 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.263315:0:18302:0:(rw26.c:858:ll_write_end()) pos 99524608, len 4096, copied 4096 00000080:00200000:1.0:1775764896.263318:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000003fb3bb8[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.263320:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000077436c57 17ffffc0001001 3:0 ffff90c212829e00 24298 lru 00000080:00200000:1.0:1775764896.263323:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d0aba9ea 24298: 1< 0 - - > 2< 99524608 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1089536 - | + - + - > 5< + - + - | 0 - | 3818 - -> 00000080:00200000:1.0:1775764896.263326:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000003fb3bb8 00000080:00200000:1.0:1775764896.263327:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.263329:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.263336:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263344:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263350:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263357:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.263368:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.263371:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.263374:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.263383:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.263385:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.263388:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.263392:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263400:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.263406:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.263408:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.263411:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.263412:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24298, 24298] 00000080:00200000:1.0:1775764896.263420:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.263422:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.263443:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99528704 i_blocks 90112 00000080:00200000:1.0:1775764896.263446:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.263461:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99528704, count: 4096 00000080:00200000:1.0:1775764896.263471:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99528704, bytes: 4096 00000080:00200000:1.0:1775764896.263473:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.263477:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24300, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.263478:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99528704, 99532800) 0 0 00000080:00200000:1.0:1775764896.263481:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.263485:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24299, 24299] 00020000:00200000:1.0:1775764896.263489:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99528704 chunk: [99528704, 99532800) 99532800, 4096 00020000:00200000:1.0:1775764896.263491:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.263503:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24299, 24299] 00000008:00010000:1.0:1775764896.263508:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.263510:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.263518:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263527:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99528704 99532799) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263536:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263544:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263551:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.263559:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99528704, 99532800) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.263564:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24299 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.263580:0:18302:0:(rw26.c:858:ll_write_end()) pos 99528704, len 4096, copied 4096 00000080:00200000:1.0:1775764896.263584:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000525d5807[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.263587:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e113cfc6 17ffffc0001001 3:0 ffff90c212829540 24299 lru 00000080:00200000:1.0:1775764896.263593:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003782db9b 24299: 1< 0 - - > 2< 99528704 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1085440 - | + - + - > 5< + - + - | 0 - | 3819 - -> 00000080:00200000:1.0:1775764896.263595:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000525d5807 00000080:00200000:1.0:1775764896.263596:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.263598:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.263606:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263613:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263620:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263633:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.263644:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.263647:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.263649:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.263658:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.263661:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.263663:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.263667:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263674:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.263683:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.263685:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.263689:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.263691:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24299, 24299] 00000080:00200000:1.0:1775764896.263702:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.263705:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.263733:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99532800 i_blocks 90112 00000080:00200000:1.0:1775764896.263737:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.263751:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99532800, count: 4096 00000080:00200000:1.0:1775764896.263762:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99532800, bytes: 4096 00000080:00200000:1.0:1775764896.263765:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.263769:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24301, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.263770:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99532800, 99536896) 0 0 00000080:00200000:1.0:1775764896.263773:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.263778:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24300, 24300] 00020000:00200000:1.0:1775764896.263781:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99532800 chunk: [99532800, 99536896) 99536896, 4096 00020000:00200000:1.0:1775764896.263783:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.263795:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24300, 24300] 00000008:00010000:1.0:1775764896.263799:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.263802:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.263809:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263818:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99532800 99536895) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263826:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263833:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.263840:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.263849:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99532800, 99536896) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.263853:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24300 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.263945:0:18302:0:(rw26.c:858:ll_write_end()) pos 99532800, len 4096, copied 4096 00000080:00200000:1.0:1775764896.263950:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c947ecf1[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.263953:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000afbd77d8 17ffffc0001021 2:0 ffff90c212829460 24300 lru 00000080:00200000:1.0:1775764896.263957:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000e0a7752 24300: 1< 0 - - > 2< 99532800 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1081344 - | + - + - > 5< + - + - | 0 - | 3820 - -> 00000080:00200000:1.0:1775764896.263960:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c947ecf1 00000080:00200000:1.0:1775764896.263961:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.263963:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.263971:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263978:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263983:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.263989:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.264000:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.264003:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.264004:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.264013:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.264016:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.264018:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.264021:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264028:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.264036:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.264038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.264042:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.264045:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24300, 24300] 00000080:00200000:1.0:1775764896.264056:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.264059:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.264088:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99536896 i_blocks 90112 00000080:00200000:1.0:1775764896.264092:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.264107:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99536896, count: 4096 00000080:00200000:1.0:1775764896.264127:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99536896, bytes: 4096 00000080:00200000:1.0:1775764896.264130:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.264134:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24302, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.264135:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99536896, 99540992) 0 0 00000080:00200000:1.0:1775764896.264138:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.264143:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24301, 24301] 00020000:00200000:1.0:1775764896.264146:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99536896 chunk: [99536896, 99540992) 99540992, 4096 00020000:00200000:1.0:1775764896.264148:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.264159:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24301, 24301] 00000008:00010000:1.0:1775764896.264164:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.264167:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.264174:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264183:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99536896 99540991) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264192:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264199:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264210:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.264216:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99536896, 99540992) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.264220:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24301 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.264240:0:18302:0:(rw26.c:858:ll_write_end()) pos 99536896, len 4096, copied 4096 00000080:00200000:1.0:1775764896.264243:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000088828474[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.264245:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006fe046cd 17ffffc0001001 3:0 ffff90c21282a000 24301 lru 00000080:00200000:1.0:1775764896.264249:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000479277d8 24301: 1< 0 - - > 2< 99536896 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1077248 - | + - + - > 5< + - + - | 0 - | 3821 - -> 00000080:00200000:1.0:1775764896.264251:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000088828474 00000080:00200000:1.0:1775764896.264251:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.264253:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.264258:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.264263:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.264268:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.264272:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.264278:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.264280:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.264282:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.264289:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.264290:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.264292:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.264295:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264300:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.264305:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.264306:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.264309:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.264310:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24301, 24301] 00000080:00200000:1.0:1775764896.264318:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.264320:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.264345:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99540992 i_blocks 90112 00000080:00200000:1.0:1775764896.264349:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.264364:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99540992, count: 4096 00000080:00200000:1.0:1775764896.264375:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99540992, bytes: 4096 00000080:00200000:1.0:1775764896.264377:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.264381:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24303, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.264382:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99540992, 99545088) 0 0 00000080:00200000:1.0:1775764896.264385:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.264390:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24302, 24302] 00020000:00200000:1.0:1775764896.264392:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99540992 chunk: [99540992, 99545088) 99545088, 4096 00020000:00200000:1.0:1775764896.264395:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.264405:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24302, 24302] 00000008:00010000:1.0:1775764896.264409:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.264411:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.264417:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264423:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99540992 99545087) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264428:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264433:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264436:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.264444:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99540992, 99545088) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.264449:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24302 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.264465:0:18302:0:(rw26.c:858:ll_write_end()) pos 99540992, len 4096, copied 4096 00000080:00200000:1.0:1775764896.264469:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d21b1456[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.264472:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000bfceb118 17ffffc0001001 3:0 ffff90c21282aee0 24302 lru 00000080:00200000:1.0:1775764896.264477:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b59bc067 24302: 1< 0 - - > 2< 99540992 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1073152 - | + - + - > 5< + - + - | 0 - | 3822 - -> 00000080:00200000:1.0:1775764896.264479:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d21b1456 00000080:00200000:1.0:1775764896.264480:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.264482:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.264489:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.264498:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.264505:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.264513:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.264524:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.264527:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.264529:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.264538:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.264541:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.264544:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.264547:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264555:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.264564:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.264566:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.264571:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.264573:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24302, 24302] 00000080:00200000:1.0:1775764896.264584:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.264587:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.264617:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99545088 i_blocks 90112 00000080:00200000:1.0:1775764896.264621:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.264637:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99545088, count: 4096 00000080:00200000:1.0:1775764896.264666:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99545088, bytes: 4096 00000080:00200000:1.0:1775764896.264669:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.264673:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24304, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.264674:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99545088, 99549184) 0 0 00000080:00200000:1.0:1775764896.264677:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.264682:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24303, 24303] 00020000:00200000:1.0:1775764896.264685:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99545088 chunk: [99545088, 99549184) 99549184, 4096 00020000:00200000:1.0:1775764896.264687:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.264699:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24303, 24303] 00000008:00010000:1.0:1775764896.264704:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.264706:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.264714:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264723:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99545088 99549183) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264731:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264739:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264745:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.264753:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99545088, 99549184) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.264758:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24303 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.264774:0:18302:0:(rw26.c:858:ll_write_end()) pos 99545088, len 4096, copied 4096 00000080:00200000:1.0:1775764896.264778:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b4bb26b6[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.264781:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000291b450a 17ffffc0001001 3:0 ffff90c21282a0e0 24303 lru 00000080:00200000:1.0:1775764896.264786:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002f28cafa 24303: 1< 0 - - > 2< 99545088 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1069056 - | + - + - > 5< + - + - | 0 - | 3823 - -> 00000080:00200000:1.0:1775764896.264788:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b4bb26b6 00000080:00200000:1.0:1775764896.264789:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.264791:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.264799:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.264806:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.264813:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.264821:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.264832:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.264835:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.264837:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.264846:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.264849:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.264851:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.264855:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.264863:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.264890:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.264893:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.264897:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.264900:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24303, 24303] 00000080:00200000:1.0:1775764896.264911:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.264914:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.264943:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99549184 i_blocks 90112 00000080:00200000:1.0:1775764896.264946:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.264962:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99549184, count: 4096 00000080:00200000:1.0:1775764896.264973:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99549184, bytes: 4096 00000080:00200000:1.0:1775764896.264976:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.264979:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24305, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.264981:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99549184, 99553280) 0 0 00000080:00200000:1.0:1775764896.264983:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.264988:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24304, 24304] 00020000:00200000:1.0:1775764896.264991:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99549184 chunk: [99549184, 99553280) 99553280, 4096 00020000:00200000:1.0:1775764896.264993:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.265005:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24304, 24304] 00000008:00010000:1.0:1775764896.265010:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.265012:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.265020:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265029:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99549184 99553279) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265037:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265045:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265049:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.265054:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99549184, 99553280) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.265058:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24304 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.265070:0:18302:0:(rw26.c:858:ll_write_end()) pos 99549184, len 4096, copied 4096 00000080:00200000:1.0:1775764896.265073:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fbae0647[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.265074:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cea5df2d 17ffffc0001001 3:0 ffff90c21282a1c0 24304 lru 00000080:00200000:1.0:1775764896.265078:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000725129e9 24304: 1< 0 - - > 2< 99549184 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1064960 - | + - + - > 5< + - + - | 0 - | 3824 - -> 00000080:00200000:1.0:1775764896.265079:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fbae0647 00000080:00200000:1.0:1775764896.265079:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.265081:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.265086:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265091:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265095:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265099:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.265109:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.265111:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.265126:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.265134:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.265136:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.265138:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.265141:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265146:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.265151:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.265152:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.265155:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.265157:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24304, 24304] 00000080:00200000:1.0:1775764896.265164:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.265166:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.265185:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99553280 i_blocks 90112 00000080:00200000:1.0:1775764896.265187:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.265198:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99553280, count: 4096 00000080:00200000:1.0:1775764896.265204:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99553280, bytes: 4096 00000080:00200000:1.0:1775764896.265206:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.265208:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24306, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.265209:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99553280, 99557376) 0 0 00000080:00200000:1.0:1775764896.265211:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.265214:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24305, 24305] 00020000:00200000:1.0:1775764896.265216:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99553280 chunk: [99553280, 99557376) 99557376, 4096 00020000:00200000:1.0:1775764896.265218:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.265224:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24305, 24305] 00000008:00010000:1.0:1775764896.265227:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.265229:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.265234:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265240:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99553280 99557375) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265246:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265251:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265254:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.265258:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99553280, 99557376) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.265261:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24305 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.265274:0:18302:0:(rw26.c:858:ll_write_end()) pos 99553280, len 4096, copied 4096 00000080:00200000:1.0:1775764896.265277:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000edd7d88c[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.265279:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e710a0b9 17ffffc0001001 3:0 ffff90c21282a7e0 24305 lru 00000080:00200000:1.0:1775764896.265282:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000064266f26 24305: 1< 0 - - > 2< 99553280 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1060864 - | + - + - > 5< + - + - | 0 - | 3825 - -> 00000080:00200000:1.0:1775764896.265284:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000edd7d88c 00000080:00200000:1.0:1775764896.265284:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.265286:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.265291:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265296:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265300:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265305:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.265311:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.265313:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.265314:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.265320:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.265321:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.265323:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.265325:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265332:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.265340:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.265342:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.265345:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.265348:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24305, 24305] 00000080:00200000:1.0:1775764896.265359:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.265361:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.265389:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99557376 i_blocks 90112 00000080:00200000:1.0:1775764896.265393:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.265408:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99557376, count: 4096 00000080:00200000:1.0:1775764896.265416:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99557376, bytes: 4096 00000080:00200000:1.0:1775764896.265417:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.265420:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24307, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.265421:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99557376, 99561472) 0 0 00000080:00200000:1.0:1775764896.265423:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.265426:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24306, 24306] 00020000:00200000:1.0:1775764896.265428:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99557376 chunk: [99557376, 99561472) 99561472, 4096 00020000:00200000:1.0:1775764896.265430:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.265437:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24306, 24306] 00000008:00010000:1.0:1775764896.265442:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.265444:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.265451:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265460:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99557376 99561471) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265468:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265475:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265482:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.265491:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99557376, 99561472) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.265496:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24306 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.265511:0:18302:0:(rw26.c:858:ll_write_end()) pos 99557376, len 4096, copied 4096 00000080:00200000:1.0:1775764896.265516:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d0d4ff74[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.265518:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000124a1717 17ffffc0001001 3:0 ffff90c21282a9a0 24306 lru 00000080:00200000:1.0:1775764896.265523:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ebe0e983 24306: 1< 0 - - > 2< 99557376 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1056768 - | + - + - > 5< + - + - | 0 - | 3826 - -> 00000080:00200000:1.0:1775764896.265526:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d0d4ff74 00000080:00200000:1.0:1775764896.265527:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.265529:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.265536:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265544:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265551:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265559:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.265570:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.265573:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.265575:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.265584:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.265586:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.265589:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.265593:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265600:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.265608:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.265611:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.265615:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.265617:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24306, 24306] 00000080:00200000:1.0:1775764896.265629:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.265632:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.265661:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99561472 i_blocks 90112 00000080:00200000:1.0:1775764896.265664:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.265680:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99561472, count: 4096 00000080:00200000:1.0:1775764896.265691:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99561472, bytes: 4096 00000080:00200000:1.0:1775764896.265693:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.265696:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24308, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.265698:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99561472, 99565568) 0 0 00000080:00200000:1.0:1775764896.265700:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.265705:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24307, 24307] 00020000:00200000:1.0:1775764896.265708:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99561472 chunk: [99561472, 99565568) 99565568, 4096 00020000:00200000:1.0:1775764896.265710:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.265721:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24307, 24307] 00000008:00010000:1.0:1775764896.265727:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.265729:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.265736:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265745:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99561472 99565567) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265754:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265761:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265768:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.265776:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99561472, 99565568) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.265781:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24307 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.265796:0:18302:0:(rw26.c:858:ll_write_end()) pos 99561472, len 4096, copied 4096 00000080:00200000:1.0:1775764896.265800:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c11b3911[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.265802:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000658caaca 17ffffc0001001 3:0 ffff90c21282a2a0 24307 lru 00000080:00200000:1.0:1775764896.265807:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000020a2190d 24307: 1< 0 - - > 2< 99561472 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1052672 - | + - + - > 5< + - + - | 0 - | 3827 - -> 00000080:00200000:1.0:1775764896.265809:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c11b3911 00000080:00200000:1.0:1775764896.265810:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.265812:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.265820:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265828:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265835:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.265847:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.265859:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.265862:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.265864:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.265968:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.265972:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.265974:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.265978:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.265987:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.265995:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.265997:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.266002:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.266005:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24307, 24307] 00000080:00200000:1.0:1775764896.266015:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.266018:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.266045:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99565568 i_blocks 90112 00000080:00200000:1.0:1775764896.266047:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.266058:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99565568, count: 4096 00000080:00200000:1.0:1775764896.266065:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99565568, bytes: 4096 00000080:00200000:1.0:1775764896.266066:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.266069:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24309, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.266070:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99565568, 99569664) 0 0 00000080:00200000:1.0:1775764896.266072:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.266076:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24308, 24308] 00020000:00200000:1.0:1775764896.266078:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99565568 chunk: [99565568, 99569664) 99569664, 4096 00020000:00200000:1.0:1775764896.266080:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.266086:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24308, 24308] 00000008:00010000:1.0:1775764896.266090:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.266091:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.266097:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266103:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99565568 99569663) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266108:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266124:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266129:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.266134:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99565568, 99569664) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.266138:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24308 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.266152:0:18302:0:(rw26.c:858:ll_write_end()) pos 99565568, len 4096, copied 4096 00000080:00200000:1.0:1775764896.266155:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a0fe2ef3[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.266157:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a587adda 17ffffc0001001 3:0 ffff90c21282ad20 24308 lru 00000080:00200000:1.0:1775764896.266160:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d426a9c6 24308: 1< 0 - - > 2< 99565568 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1048576 - | + - + - > 5< + - + - | 0 - | 3828 - -> 00000080:00200000:1.0:1775764896.266162:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a0fe2ef3 00000080:00200000:1.0:1775764896.266163:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.266164:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.266170:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.266174:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.266178:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.266183:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.266189:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.266191:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.266192:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.266199:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.266200:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.266202:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.266204:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266210:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.266215:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.266216:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.266219:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.266221:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24308, 24308] 00000080:00200000:1.0:1775764896.266228:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.266230:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.266248:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99569664 i_blocks 90112 00000080:00200000:1.0:1775764896.266250:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.266261:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99569664, count: 4096 00000080:00200000:1.0:1775764896.266267:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99569664, bytes: 4096 00000080:00200000:1.0:1775764896.266268:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.266271:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24310, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.266272:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99569664, 99573760) 0 0 00000080:00200000:1.0:1775764896.266273:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.266277:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24309, 24309] 00020000:00200000:1.0:1775764896.266279:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99569664 chunk: [99569664, 99573760) 99573760, 4096 00020000:00200000:1.0:1775764896.266280:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.266286:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24309, 24309] 00000008:00010000:1.0:1775764896.266289:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.266291:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.266296:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266302:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99569664 99573759) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266308:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266312:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266320:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.266325:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99569664, 99573760) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.266329:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24309 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.266344:0:18302:0:(rw26.c:858:ll_write_end()) pos 99569664, len 4096, copied 4096 00000080:00200000:1.0:1775764896.266348:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001ee26baa[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.266350:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000df47ab2a 17ffffc0001001 3:0 ffff90c21282a380 24309 lru 00000080:00200000:1.0:1775764896.266355:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000589770a5 24309: 1< 0 - - > 2< 99569664 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1044480 - | + - + - > 5< + - + - | 0 - | 3829 - -> 00000080:00200000:1.0:1775764896.266357:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001ee26baa 00000080:00200000:1.0:1775764896.266358:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.266360:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.266367:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.266374:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.266381:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.266389:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.266400:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.266403:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.266406:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.266413:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.266415:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.266417:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.266419:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266425:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.266430:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.266431:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.266434:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.266435:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24309, 24309] 00000080:00200000:1.0:1775764896.266445:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.266449:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.266476:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99573760 i_blocks 90112 00000080:00200000:1.0:1775764896.266479:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.266495:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99573760, count: 4096 00000080:00200000:1.0:1775764896.266506:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99573760, bytes: 4096 00000080:00200000:1.0:1775764896.266508:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.266512:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24311, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.266514:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99573760, 99577856) 0 0 00000080:00200000:1.0:1775764896.266516:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.266521:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24310, 24310] 00020000:00200000:1.0:1775764896.266524:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99573760 chunk: [99573760, 99577856) 99577856, 4096 00020000:00200000:1.0:1775764896.266526:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.266537:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24310, 24310] 00000008:00010000:1.0:1775764896.266542:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.266545:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.266553:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266561:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99573760 99577855) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266570:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266578:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266584:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.266591:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99573760, 99577856) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.266596:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24310 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.266612:0:18302:0:(rw26.c:858:ll_write_end()) pos 99573760, len 4096, copied 4096 00000080:00200000:1.0:1775764896.266616:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007e03e57a[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.266619:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002fb63124 17ffffc0001001 3:0 ffff90c21282a620 24310 lru 00000080:00200000:1.0:1775764896.266624:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004b956238 24310: 1< 0 - - > 2< 99573760 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1040384 - | + - + - > 5< + - + - | 0 - | 3830 - -> 00000080:00200000:1.0:1775764896.266626:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007e03e57a 00000080:00200000:1.0:1775764896.266627:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.266629:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.266637:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.266645:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.266653:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.266660:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.266671:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.266674:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.266676:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.266685:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.266688:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.266690:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.266694:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266701:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.266709:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.266711:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.266715:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.266718:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24310, 24310] 00000080:00200000:1.0:1775764896.266729:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.266733:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.266762:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99577856 i_blocks 90112 00000080:00200000:1.0:1775764896.266765:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.266781:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99577856, count: 4096 00000080:00200000:1.0:1775764896.266791:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99577856, bytes: 4096 00000080:00200000:1.0:1775764896.266794:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.266797:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24312, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.266799:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99577856, 99581952) 0 0 00000080:00200000:1.0:1775764896.266801:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.266806:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24311, 24311] 00020000:00200000:1.0:1775764896.266809:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99577856 chunk: [99577856, 99581952) 99581952, 4096 00020000:00200000:1.0:1775764896.266812:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.266823:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24311, 24311] 00000008:00010000:1.0:1775764896.266828:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.266831:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.266839:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266847:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99577856 99581951) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266855:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266861:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.266950:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.266961:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99577856, 99581952) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.266967:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24311 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.266984:0:18302:0:(rw26.c:858:ll_write_end()) pos 99577856, len 4096, copied 4096 00000080:00200000:1.0:1775764896.266988:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ccf19373[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.266991:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005062a7c7 17ffffc0001001 3:0 ffff90c21282ae00 24311 lru 00000080:00200000:1.0:1775764896.266996:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d063c43d 24311: 1< 0 - - > 2< 99577856 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1036288 - | + - + - > 5< + - + - | 0 - | 3831 - -> 00000080:00200000:1.0:1775764896.266999:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ccf19373 00000080:00200000:1.0:1775764896.267000:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.267002:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.267009:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.267017:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.267024:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.267032:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.270260:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.270268:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.270272:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.270297:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.270299:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.270302:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.270309:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270320:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.270332:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.270335:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.270343:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.270351:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24311, 24311] 00000080:00200000:1.0:1775764896.270382:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.270387:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.270435:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99581952 i_blocks 90112 00000080:00200000:1.0:1775764896.270441:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.270472:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99581952, count: 4096 00000080:00200000:1.0:1775764896.270488:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99581952, bytes: 4096 00000080:00200000:1.0:1775764896.270492:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.270498:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24313, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.270500:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99581952, 99586048) 0 0 00000080:00200000:1.0:1775764896.270504:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.270516:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24312, 24312] 00020000:00200000:1.0:1775764896.270521:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99581952 chunk: [99581952, 99586048) 99586048, 4096 00020000:00200000:1.0:1775764896.270525:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.270542:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24312, 24312] 00000008:00010000:1.0:1775764896.270553:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.270557:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.270574:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270586:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99581952 99586047) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270596:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270610:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270617:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.270627:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99581952, 99586048) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.270637:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24312 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.270676:0:18302:0:(rw26.c:858:ll_write_end()) pos 99581952, len 4096, copied 4096 00000080:00200000:1.0:1775764896.270682:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000061ec5d28[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.270686:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009f68f533 17ffffc0001001 3:0 ffff90c21282a540 24312 lru 00000080:00200000:1.0:1775764896.270692:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f74c7982 24312: 1< 0 - - > 2< 99581952 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1032192 - | + - + - > 5< + - + - | 0 - | 3832 - -> 00000080:00200000:1.0:1775764896.270694:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000061ec5d28 00000080:00200000:1.0:1775764896.270695:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.270698:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.270711:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.270721:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.270728:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.270736:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.270753:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.270756:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.270758:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.270768:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.270771:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.270773:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.270778:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270786:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.270795:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.270797:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.270801:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.270804:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24312, 24312] 00000080:00200000:1.0:1775764896.270815:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.270819:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.270848:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99586048 i_blocks 90112 00000080:00200000:1.0:1775764896.270852:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.270895:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99586048, count: 4096 00000080:00200000:1.0:1775764896.270909:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99586048, bytes: 4096 00000080:00200000:1.0:1775764896.270911:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.270915:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24314, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.270917:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99586048, 99590144) 0 0 00000080:00200000:1.0:1775764896.270919:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.270925:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24313, 24313] 00020000:00200000:1.0:1775764896.270928:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99586048 chunk: [99586048, 99590144) 99590144, 4096 00020000:00200000:1.0:1775764896.270930:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.270941:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24313, 24313] 00000008:00010000:1.0:1775764896.270947:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.270949:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.270957:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270966:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99586048 99590143) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270975:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270982:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.270988:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.270996:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99586048, 99590144) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.271002:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24313 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.271018:0:18302:0:(rw26.c:858:ll_write_end()) pos 99586048, len 4096, copied 4096 00000080:00200000:1.0:1775764896.271022:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000082bc463f[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.271025:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008340ef7d 17ffffc0001001 3:0 ffff90c21282a460 24313 lru 00000080:00200000:1.0:1775764896.271030:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000039eb5c3e 24313: 1< 0 - - > 2< 99586048 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1028096 - | + - + - > 5< + - + - | 0 - | 3833 - -> 00000080:00200000:1.0:1775764896.271032:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000082bc463f 00000080:00200000:1.0:1775764896.271033:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.271034:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.271041:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271046:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271050:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271054:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.271061:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.271063:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.271064:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.271071:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.271073:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.271074:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.271077:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271082:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.271087:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.271088:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.271091:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.271093:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24313, 24313] 00000080:00200000:1.0:1775764896.271101:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.271103:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.271136:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99590144 i_blocks 90112 00000080:00200000:1.0:1775764896.271139:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.271150:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99590144, count: 4096 00000080:00200000:1.0:1775764896.271156:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99590144, bytes: 4096 00000080:00200000:1.0:1775764896.271158:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.271161:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24315, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.271162:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99590144, 99594240) 0 0 00000080:00200000:1.0:1775764896.271163:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.271167:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24314, 24314] 00020000:00200000:1.0:1775764896.271170:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99590144 chunk: [99590144, 99594240) 99594240, 4096 00020000:00200000:1.0:1775764896.271171:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.271178:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24314, 24314] 00000008:00010000:1.0:1775764896.271182:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.271184:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.271189:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271195:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99590144 99594239) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271201:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271205:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271209:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.271215:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99590144, 99594240) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.271218:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24314 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.271231:0:18302:0:(rw26.c:858:ll_write_end()) pos 99590144, len 4096, copied 4096 00000080:00200000:1.0:1775764896.271234:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ce1dc570[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.271236:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000429877ef 17ffffc0001001 3:0 ffff90c21282aa80 24314 lru 00000080:00200000:1.0:1775764896.271240:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cd7c1f8e 24314: 1< 0 - - > 2< 99590144 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1024000 - | + - + - > 5< + - + - | 0 - | 3834 - -> 00000080:00200000:1.0:1775764896.271241:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ce1dc570 00000080:00200000:1.0:1775764896.271242:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.271243:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.271248:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271253:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271257:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271261:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.271268:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.271270:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.271271:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.271277:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.271279:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.271280:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.271283:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271288:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.271294:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.271295:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.271298:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.271300:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24314, 24314] 00000080:00200000:1.0:1775764896.271307:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.271309:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.271328:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99594240 i_blocks 90112 00000080:00200000:1.0:1775764896.271332:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.271346:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99594240, count: 4096 00000080:00200000:1.0:1775764896.271356:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99594240, bytes: 4096 00000080:00200000:1.0:1775764896.271359:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.271362:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24316, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.271364:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99594240, 99598336) 0 0 00000080:00200000:1.0:1775764896.271366:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.271371:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24315, 24315] 00020000:00200000:1.0:1775764896.271374:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99594240 chunk: [99594240, 99598336) 99598336, 4096 00020000:00200000:1.0:1775764896.271377:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.271388:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24315, 24315] 00000008:00010000:1.0:1775764896.271393:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.271396:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.271403:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271410:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99594240 99598335) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271417:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271421:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271425:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.271430:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99594240, 99598336) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.271434:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24315 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.271454:0:18302:0:(rw26.c:858:ll_write_end()) pos 99594240, len 4096, copied 4096 00000080:00200000:1.0:1775764896.271459:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e9c316df[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.271462:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005767f4ee 17ffffc0001021 2:0 ffff90c21282a700 24315 lru 00000080:00200000:1.0:1775764896.271467:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005a792d14 24315: 1< 0 - - > 2< 99594240 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1019904 - | + - + - > 5< + - + - | 0 - | 3835 - -> 00000080:00200000:1.0:1775764896.271469:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e9c316df 00000080:00200000:1.0:1775764896.271470:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.271472:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.271480:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271488:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271495:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271508:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.271519:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.271522:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.271524:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.271533:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.271536:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.271538:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.271542:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271550:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.271559:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.271562:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.271566:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.271569:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24315, 24315] 00000080:00200000:1.0:1775764896.271580:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.271583:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.271612:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99598336 i_blocks 90112 00000080:00200000:1.0:1775764896.271615:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.271632:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99598336, count: 4096 00000080:00200000:1.0:1775764896.271643:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99598336, bytes: 4096 00000080:00200000:1.0:1775764896.271645:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.271650:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24317, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.271651:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99598336, 99602432) 0 0 00000080:00200000:1.0:1775764896.271653:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.271659:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24316, 24316] 00020000:00200000:1.0:1775764896.271662:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99598336 chunk: [99598336, 99602432) 99602432, 4096 00020000:00200000:1.0:1775764896.271664:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.271676:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24316, 24316] 00000008:00010000:1.0:1775764896.271681:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.271684:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.271692:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271701:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99598336 99602431) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271709:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271716:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271723:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.271731:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99598336, 99602432) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.271737:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24316 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.271752:0:18302:0:(rw26.c:858:ll_write_end()) pos 99598336, len 4096, copied 4096 00000080:00200000:1.0:1775764896.271757:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000087e46b28[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.271760:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b2e52a70 17ffffc0001001 3:0 ffff90c21282ac40 24316 lru 00000080:00200000:1.0:1775764896.271765:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004a3f1b0e 24316: 1< 0 - - > 2< 99598336 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1015808 - | + - + - > 5< + - + - | 0 - | 3836 - -> 00000080:00200000:1.0:1775764896.271768:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000087e46b28 00000080:00200000:1.0:1775764896.271769:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.271771:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.271778:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271786:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271792:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.271799:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.271811:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.271814:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.271817:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.271826:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.271828:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.271831:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.271834:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.271842:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.271850:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.271852:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.271856:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.271859:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24316, 24316] 00000080:00200000:1.0:1775764896.271894:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.271897:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.271927:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99602432 i_blocks 90112 00000080:00200000:1.0:1775764896.271931:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.271946:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99602432, count: 4096 00000080:00200000:1.0:1775764896.271957:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99602432, bytes: 4096 00000080:00200000:1.0:1775764896.271960:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.271963:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24318, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.271965:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99602432, 99606528) 0 0 00000080:00200000:1.0:1775764896.271967:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.271973:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24317, 24317] 00020000:00200000:1.0:1775764896.271976:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99602432 chunk: [99602432, 99606528) 99606528, 4096 00020000:00200000:1.0:1775764896.271978:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.271990:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24317, 24317] 00000008:00010000:1.0:1775764896.271995:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.271998:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.272005:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.272014:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99602432 99606527) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.272023:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.272030:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.272039:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.272044:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99602432, 99606528) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.272048:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24317 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.272061:0:18302:0:(rw26.c:858:ll_write_end()) pos 99602432, len 4096, copied 4096 00000080:00200000:1.0:1775764896.272064:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000043b7b339[3 00000000b99c1f5d 2 1 00000000dea68d45] 00000080:00200000:1.0:1775764896.272066:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000091a73419 17ffffc0001001 3:0 ffff90c21282a8c0 24317 lru 00000080:00200000:1.0:1775764896.272069:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000033973a01 24317: 1< 0 - - > 2< 99602432 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1011712 - | + - + - > 5< + - + - | 0 - | 3837 - -> 00000080:00200000:1.0:1775764896.272071:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000043b7b339 00000080:00200000:1.0:1775764896.272072:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.272073:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.272078:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.272083:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000004a837112 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.272087:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.272091:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.272098:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.272100:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.272101:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.272109:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.272111:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.272131:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eeb48/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.272135:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.272143:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.272151:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195eaf38 (ffff90c2195ee908/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.272153:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee908/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.272158:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.272160:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24317, 24317] 00000080:00200000:1.0:1775764896.272171:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.272174:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.272222:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99606528 i_blocks 90112 00000080:00200000:0.0:1775764896.272227:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.272247:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99606528, count: 4096 00000080:00200000:0.0:1775764896.272264:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99606528, bytes: 4096 00000080:00200000:0.0:1775764896.272267:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.272271:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24319, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.272272:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99606528, 99610624) 0 0 00000080:00200000:0.0:1775764896.272274:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.272278:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24318, 24318] 00020000:00200000:0.0:1775764896.272280:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99606528 chunk: [99606528, 99610624) 99610624, 4096 00020000:00200000:0.0:1775764896.272281:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.272293:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24318, 24318] 00000008:00010000:0.0:1775764896.272298:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.272300:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.272306:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272312:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99606528 99610623) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272317:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272322:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272340:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.272345:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99606528, 99610624) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.272348:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24318 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.272361:0:18302:0:(rw26.c:858:ll_write_end()) pos 99606528, len 4096, copied 4096 00000080:00200000:0.0:1775764896.272364:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000085c9f384[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.272366:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000afb95956 17ffffc0001001 3:0 ffff90c21282d000 24318 lru 00000080:00200000:0.0:1775764896.272370:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000012680cf3 24318: 1< 0 - - > 2< 99606528 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1007616 - | + - + - > 5< + - + - | 0 - | 3838 - -> 00000080:00200000:0.0:1775764896.272371:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000085c9f384 00000080:00200000:0.0:1775764896.272372:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.272374:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.272379:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.272385:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.272389:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.272393:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.272400:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.272401:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.272403:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.272412:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.272414:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.272415:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.272418:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272423:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.272428:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.272430:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.272434:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.272436:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24318, 24318] 00000080:00200000:0.0:1775764896.272459:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.272461:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.272488:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99610624 i_blocks 90112 00000080:00200000:0.0:1775764896.272491:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.272503:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99610624, count: 4096 00000080:00200000:0.0:1775764896.272515:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99610624, bytes: 4096 00000080:00200000:0.0:1775764896.272517:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.272519:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24320, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.272520:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99610624, 99614720) 0 0 00000080:00200000:0.0:1775764896.272522:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.272526:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24319, 24319] 00020000:00200000:0.0:1775764896.272527:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99610624 chunk: [99610624, 99614720) 99614720, 4096 00020000:00200000:0.0:1775764896.272529:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.272535:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24319, 24319] 00000008:00010000:0.0:1775764896.272544:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.272546:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.272552:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272558:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99610624 99614719) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272563:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272568:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272571:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.272576:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99610624, 99614720) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.272579:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24319 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.272591:0:18302:0:(rw26.c:858:ll_write_end()) pos 99610624, len 4096, copied 4096 00000080:00200000:0.0:1775764896.272602:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d76d27f0[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.272604:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005985a93d 17ffffc0001001 3:0 ffff90c21282dee0 24319 lru 00000080:00200000:0.0:1775764896.272608:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003235205c 24319: 1< 0 - - > 2< 99610624 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 1003520 - | + - + - > 5< + - + - | 0 - | 3839 - -> 00000080:00200000:0.0:1775764896.272609:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d76d27f0 00000080:00200000:0.0:1775764896.272610:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.272611:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.272617:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.272626:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.272631:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.272636:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.272643:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.272644:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.272646:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.272652:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.272654:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.272655:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.272658:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.272663:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.272668:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.272669:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.273177:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.273180:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24319, 24319] 00000080:00200000:0.0:1775764896.273189:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.273191:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.273216:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99614720 i_blocks 90112 00000080:00200000:0.0:1775764896.273218:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.273230:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99614720, count: 4096 00000080:00200000:0.0:1775764896.273237:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99614720, bytes: 4096 00000080:00200000:0.0:1775764896.273239:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.273242:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24321, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.273243:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99614720, 99618816) 0 0 00000080:00200000:0.0:1775764896.273244:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.273248:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24320, 24320] 00020000:00200000:0.0:1775764896.273251:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99614720 chunk: [99614720, 99618816) 99618816, 4096 00020000:00200000:0.0:1775764896.273252:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.273260:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24320, 24320] 00000008:00010000:0.0:1775764896.273264:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.273266:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.273272:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273278:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99614720 99618815) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273284:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273289:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273292:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.273297:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99614720, 99618816) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.273301:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24320 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.273320:0:18302:0:(rw26.c:858:ll_write_end()) pos 99614720, len 4096, copied 4096 00000080:00200000:0.0:1775764896.273324:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000991ca649[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.273326:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a56db1fd 17ffffc0001001 3:0 ffff90c21282d0e0 24320 lru 00000080:00200000:0.0:1775764896.273332:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007e238ef5 24320: 1< 0 - - > 2< 99614720 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 999424 - | + - + - > 5< + - + - | 0 - | 3840 - -> 00000080:00200000:0.0:1775764896.273334:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000991ca649 00000080:00200000:0.0:1775764896.273335:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.273337:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.273346:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.273353:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.273360:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.273368:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.273381:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.273383:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.273386:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.273396:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.273398:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.273400:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.273403:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273408:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.273414:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.273416:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.273419:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.273421:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24320, 24320] 00000080:00200000:0.0:1775764896.273428:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.273430:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.273456:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99618816 i_blocks 90112 00000080:00200000:0.0:1775764896.273460:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.273476:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99618816, count: 4096 00000080:00200000:0.0:1775764896.273486:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99618816, bytes: 4096 00000080:00200000:0.0:1775764896.273489:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.273493:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24322, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.273494:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99618816, 99622912) 0 0 00000080:00200000:0.0:1775764896.273497:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.273502:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24321, 24321] 00020000:00200000:0.0:1775764896.273505:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99618816 chunk: [99618816, 99622912) 99622912, 4096 00020000:00200000:0.0:1775764896.273507:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.273518:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24321, 24321] 00000008:00010000:0.0:1775764896.273524:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.273526:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.273534:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273543:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99618816 99622911) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273552:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273564:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273571:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.273579:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99618816, 99622912) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.273585:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24321 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.273601:0:18302:0:(rw26.c:858:ll_write_end()) pos 99618816, len 4096, copied 4096 00000080:00200000:0.0:1775764896.273605:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fd475fbc[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.273608:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000918f6bae 17ffffc0001001 3:0 ffff90c21282d1c0 24321 lru 00000080:00200000:0.0:1775764896.273613:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000048af5692 24321: 1< 0 - - > 2< 99618816 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 995328 - | + - + - > 5< + - + - | 0 - | 3841 - -> 00000080:00200000:0.0:1775764896.273615:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fd475fbc 00000080:00200000:0.0:1775764896.273616:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.273619:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.273626:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.273634:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.273641:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.273648:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.273665:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.273668:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.273671:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.273680:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.273683:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.273685:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.273689:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273696:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.273704:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.273707:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.273711:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.273714:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24321, 24321] 00000080:00200000:0.0:1775764896.273725:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.273729:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.273758:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99622912 i_blocks 90112 00000080:00200000:0.0:1775764896.273762:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.273778:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99622912, count: 4096 00000080:00200000:0.0:1775764896.273789:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99622912, bytes: 4096 00000080:00200000:0.0:1775764896.273791:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.273795:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24323, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.273796:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99622912, 99627008) 0 0 00000080:00200000:0.0:1775764896.273799:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.273804:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24322, 24322] 00020000:00200000:0.0:1775764896.273807:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99622912 chunk: [99622912, 99627008) 99627008, 4096 00020000:00200000:0.0:1775764896.273809:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.273821:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24322, 24322] 00000008:00010000:0.0:1775764896.273826:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.273828:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.273836:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273845:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99622912 99627007) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273853:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273860:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.273867:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.273913:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99622912, 99627008) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.273918:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24322 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.273947:0:18302:0:(rw26.c:858:ll_write_end()) pos 99622912, len 4096, copied 4096 00000080:00200000:0.0:1775764896.273951:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d3dfef95[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.273954:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000981f9823 17ffffc0001001 3:0 ffff90c21282e380 24322 lru 00000080:00200000:0.0:1775764896.273959:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000812a1c48 24322: 1< 0 - - > 2< 99622912 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 991232 - | + - + - > 5< + - + - | 0 - | 3842 - -> 00000080:00200000:0.0:1775764896.273961:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d3dfef95 00000080:00200000:0.0:1775764896.273962:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.273965:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.273973:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.273981:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.273988:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.273996:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.274005:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.274008:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.274009:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.274017:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.274019:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.274020:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.274023:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274028:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.274033:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.274034:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.274037:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.274039:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24322, 24322] 00000080:00200000:0.0:1775764896.274047:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.274049:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.274068:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99627008 i_blocks 90112 00000080:00200000:0.0:1775764896.274070:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.274081:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99627008, count: 4096 00000080:00200000:0.0:1775764896.274087:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99627008, bytes: 4096 00000080:00200000:0.0:1775764896.274089:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.274091:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24324, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.274092:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99627008, 99631104) 0 0 00000080:00200000:0.0:1775764896.274094:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.274097:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24323, 24323] 00020000:00200000:0.0:1775764896.274099:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99627008 chunk: [99627008, 99631104) 99631104, 4096 00020000:00200000:0.0:1775764896.274101:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.274107:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24323, 24323] 00000008:00010000:0.0:1775764896.274111:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.274146:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.274154:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274163:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99627008 99631103) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274170:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274177:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274183:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.274192:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99627008, 99631104) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.274196:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24323 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.274209:0:18302:0:(rw26.c:858:ll_write_end()) pos 99627008, len 4096, copied 4096 00000080:00200000:0.0:1775764896.274212:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004ea7cec3[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.274214:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000044812832 17ffffc0001001 3:0 ffff90c21282e620 24323 lru 00000080:00200000:0.0:1775764896.274218:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d2c229ad 24323: 1< 0 - - > 2< 99627008 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 987136 - | + - + - > 5< + - + - | 0 - | 3843 - -> 00000080:00200000:0.0:1775764896.274219:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004ea7cec3 00000080:00200000:0.0:1775764896.274220:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.274221:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.274227:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.274231:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.274236:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.274240:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.274247:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.274248:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.274250:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.274257:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.274258:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.274260:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.274263:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274268:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.274273:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.274275:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.274278:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.274280:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24323, 24323] 00000080:00200000:0.0:1775764896.274287:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.274289:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.274308:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99631104 i_blocks 90112 00000080:00200000:0.0:1775764896.274310:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.274321:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99631104, count: 4096 00000080:00200000:0.0:1775764896.274329:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99631104, bytes: 4096 00000080:00200000:0.0:1775764896.274332:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.274335:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24325, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.274337:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99631104, 99635200) 0 0 00000080:00200000:0.0:1775764896.274339:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.274343:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24324, 24324] 00020000:00200000:0.0:1775764896.274346:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99631104 chunk: [99631104, 99635200) 99635200, 4096 00020000:00200000:0.0:1775764896.274348:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.274359:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24324, 24324] 00000008:00010000:0.0:1775764896.274364:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.274367:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.274374:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274382:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99631104 99635199) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274392:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274399:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274405:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.274411:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99631104, 99635200) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.274415:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24324 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.274427:0:18302:0:(rw26.c:858:ll_write_end()) pos 99631104, len 4096, copied 4096 00000080:00200000:0.0:1775764896.274430:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f96642e6[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.274432:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001f328160 17ffffc0001001 3:0 ffff90c21282ee00 24324 lru 00000080:00200000:0.0:1775764896.274435:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002291fdf8 24324: 1< 0 - - > 2< 99631104 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 983040 - | + - + - > 5< + - + - | 0 - | 3844 - -> 00000080:00200000:0.0:1775764896.274437:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f96642e6 00000080:00200000:0.0:1775764896.274438:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.274439:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.274446:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.274454:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.274460:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.274472:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.274484:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.274487:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.274490:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.274499:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.274502:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.274504:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.274508:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274516:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.274524:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.274526:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.274530:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.274533:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24324, 24324] 00000080:00200000:0.0:1775764896.274544:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.274547:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.274577:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99635200 i_blocks 90112 00000080:00200000:0.0:1775764896.274581:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.274596:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99635200, count: 4096 00000080:00200000:0.0:1775764896.274607:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99635200, bytes: 4096 00000080:00200000:0.0:1775764896.274609:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.274613:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24326, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.274615:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99635200, 99639296) 0 0 00000080:00200000:0.0:1775764896.274618:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.274623:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24325, 24325] 00020000:00200000:0.0:1775764896.274626:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99635200 chunk: [99635200, 99639296) 99639296, 4096 00020000:00200000:0.0:1775764896.274628:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.274640:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24325, 24325] 00000008:00010000:0.0:1775764896.274645:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.274648:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.274655:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274664:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99635200 99639295) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274672:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274680:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274687:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.274694:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99635200, 99639296) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.274700:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24325 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.274716:0:18302:0:(rw26.c:858:ll_write_end()) pos 99635200, len 4096, copied 4096 00000080:00200000:0.0:1775764896.274720:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000b558531[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.274723:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fb768d03 17ffffc0001001 3:0 ffff90c21282e540 24325 lru 00000080:00200000:0.0:1775764896.274729:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000eb7139d5 24325: 1< 0 - - > 2< 99635200 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 978944 - | + - + - > 5< + - + - | 0 - | 3845 - -> 00000080:00200000:0.0:1775764896.274732:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000b558531 00000080:00200000:0.0:1775764896.274733:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.274735:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.274742:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.274750:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.274757:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.274764:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.274776:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.274778:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.274781:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.274790:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.274793:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.274796:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.274799:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274807:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.274816:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.274818:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.274822:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.274825:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24325, 24325] 00000080:00200000:0.0:1775764896.274836:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.274839:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.274900:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99639296 i_blocks 90112 00000080:00200000:0.0:1775764896.274904:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.274922:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99639296, count: 4096 00000080:00200000:0.0:1775764896.274933:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99639296, bytes: 4096 00000080:00200000:0.0:1775764896.274936:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.274940:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24327, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.274941:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99639296, 99643392) 0 0 00000080:00200000:0.0:1775764896.274944:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.274949:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24326, 24326] 00020000:00200000:0.0:1775764896.274952:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99639296 chunk: [99639296, 99643392) 99643392, 4096 00020000:00200000:0.0:1775764896.274954:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.274965:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24326, 24326] 00000008:00010000:0.0:1775764896.274970:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.274972:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.274980:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274989:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99639296 99643391) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.274998:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275006:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275017:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.275022:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99639296, 99643392) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.275026:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24326 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.275044:0:18302:0:(rw26.c:858:ll_write_end()) pos 99639296, len 4096, copied 4096 00000080:00200000:0.0:1775764896.275047:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004985c19f[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.275049:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000036776065 17ffffc0001021 2:0 ffff90c21282e460 24326 lru 00000080:00200000:0.0:1775764896.275053:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000073b7b716 24326: 1< 0 - - > 2< 99639296 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 974848 - | + - + - > 5< + - + - | 0 - | 3846 - -> 00000080:00200000:0.0:1775764896.275054:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004985c19f 00000080:00200000:0.0:1775764896.275055:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.275057:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.275062:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275067:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275071:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275075:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.275082:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.275084:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.275085:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.275092:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.275094:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.275096:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.275099:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275104:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.275109:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.275110:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.275125:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.275128:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24326, 24326] 00000080:00200000:0.0:1775764896.275135:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.275137:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.275156:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99643392 i_blocks 90112 00000080:00200000:0.0:1775764896.275159:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.275170:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99643392, count: 4096 00000080:00200000:0.0:1775764896.275176:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99643392, bytes: 4096 00000080:00200000:0.0:1775764896.275177:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.275180:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24328, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.275181:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99643392, 99647488) 0 0 00000080:00200000:0.0:1775764896.275183:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.275186:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24327, 24327] 00020000:00200000:0.0:1775764896.275188:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99643392 chunk: [99643392, 99647488) 99647488, 4096 00020000:00200000:0.0:1775764896.275190:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.275196:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24327, 24327] 00000008:00010000:0.0:1775764896.275199:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.275201:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.275207:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275212:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99643392 99647487) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275218:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275222:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275226:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.275230:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99643392, 99647488) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.275233:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24327 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.275245:0:18302:0:(rw26.c:858:ll_write_end()) pos 99643392, len 4096, copied 4096 00000080:00200000:0.0:1775764896.275248:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f5dc8b98[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.275250:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d6c367d7 17ffffc0001001 3:0 ffff90c21282ea80 24327 lru 00000080:00200000:0.0:1775764896.275253:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003d6a1582 24327: 1< 0 - - > 2< 99643392 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 970752 - | + - + - > 5< + - + - | 0 - | 3847 - -> 00000080:00200000:0.0:1775764896.275255:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f5dc8b98 00000080:00200000:0.0:1775764896.275256:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.275257:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.275263:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275267:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275272:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275276:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.275283:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.275285:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.275286:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.275292:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.275294:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.275295:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.275297:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275303:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.275307:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.275309:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.275311:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.275313:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24327, 24327] 00000080:00200000:0.0:1775764896.275320:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.275322:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.275347:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99647488 i_blocks 90112 00000080:00200000:0.0:1775764896.275351:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.275366:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99647488, count: 4096 00000080:00200000:0.0:1775764896.275376:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99647488, bytes: 4096 00000080:00200000:0.0:1775764896.275378:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.275382:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24329, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.275384:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99647488, 99651584) 0 0 00000080:00200000:0.0:1775764896.275386:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.275392:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24328, 24328] 00020000:00200000:0.0:1775764896.275395:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99647488 chunk: [99647488, 99651584) 99651584, 4096 00020000:00200000:0.0:1775764896.275397:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.275408:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24328, 24328] 00000008:00010000:0.0:1775764896.275412:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.275414:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.275419:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275425:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99647488 99651583) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275431:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275435:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275439:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.275444:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99647488, 99651584) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.275448:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24328 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.275465:0:18302:0:(rw26.c:858:ll_write_end()) pos 99647488, len 4096, copied 4096 00000080:00200000:0.0:1775764896.275469:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002347f147[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.275472:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c768202d 17ffffc0001001 3:0 ffff90c21282e700 24328 lru 00000080:00200000:0.0:1775764896.275477:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e4db065c 24328: 1< 0 - - > 2< 99647488 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 966656 - | + - + - > 5< + - + - | 0 - | 3848 - -> 00000080:00200000:0.0:1775764896.275479:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002347f147 00000080:00200000:0.0:1775764896.275480:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.275482:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.275490:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275498:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275505:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275513:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.275524:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.275527:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.275530:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.275538:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.275541:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.275544:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.275548:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275556:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.275565:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.275567:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.275571:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.275574:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24328, 24328] 00000080:00200000:0.0:1775764896.275584:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.275587:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.275615:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99651584 i_blocks 90112 00000080:00200000:0.0:1775764896.275619:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.275634:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99651584, count: 4096 00000080:00200000:0.0:1775764896.275645:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99651584, bytes: 4096 00000080:00200000:0.0:1775764896.275648:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.275651:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24330, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.275653:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99651584, 99655680) 0 0 00000080:00200000:0.0:1775764896.275655:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.275660:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24329, 24329] 00020000:00200000:0.0:1775764896.275663:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99651584 chunk: [99651584, 99655680) 99655680, 4096 00020000:00200000:0.0:1775764896.275665:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.275676:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24329, 24329] 00000008:00010000:0.0:1775764896.275682:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.275684:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.275691:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275700:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99651584 99655679) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275708:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275722:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275729:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.275736:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99651584, 99655680) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.275741:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24329 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.275756:0:18302:0:(rw26.c:858:ll_write_end()) pos 99651584, len 4096, copied 4096 00000080:00200000:0.0:1775764896.275760:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000057c1dd03[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.275763:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000061af1952 17ffffc0001001 3:0 ffff90c21282ec40 24329 lru 00000080:00200000:0.0:1775764896.275768:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000089db942f 24329: 1< 0 - - > 2< 99651584 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 962560 - | + - + - > 5< + - + - | 0 - | 3849 - -> 00000080:00200000:0.0:1775764896.275770:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000057c1dd03 00000080:00200000:0.0:1775764896.275771:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.275773:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.275781:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275789:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275795:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.275803:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.275818:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.275821:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.275823:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.275833:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.275835:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.275838:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.275842:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.275850:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.275857:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.275860:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.275864:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.275866:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24329, 24329] 00000080:00200000:0.0:1775764896.275900:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.275903:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.275933:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99655680 i_blocks 90112 00000080:00200000:0.0:1775764896.275936:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.275951:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99655680, count: 4096 00000080:00200000:0.0:1775764896.275961:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99655680, bytes: 4096 00000080:00200000:0.0:1775764896.275964:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.275967:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24331, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.275969:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99655680, 99659776) 0 0 00000080:00200000:0.0:1775764896.275971:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.275977:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24330, 24330] 00020000:00200000:0.0:1775764896.275980:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99655680 chunk: [99655680, 99659776) 99659776, 4096 00020000:00200000:0.0:1775764896.275982:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.275996:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24330, 24330] 00000008:00010000:0.0:1775764896.276001:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.276004:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.276011:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276018:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99655680 99659775) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276026:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276032:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276039:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.276047:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99655680, 99659776) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.276052:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24330 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.276067:0:18302:0:(rw26.c:858:ll_write_end()) pos 99655680, len 4096, copied 4096 00000080:00200000:0.0:1775764896.276071:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000094c74c94[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.276074:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005a76f6f3 17ffffc0001001 3:0 ffff90c21282e8c0 24330 lru 00000080:00200000:0.0:1775764896.276080:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003baa3a9d 24330: 1< 0 - - > 2< 99655680 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 958464 - | + - + - > 5< + - + - | 0 - | 3850 - -> 00000080:00200000:0.0:1775764896.276082:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000094c74c94 00000080:00200000:0.0:1775764896.276083:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.276085:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.276093:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.276101:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.276108:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.276149:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.276162:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.276166:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.276168:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.276177:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.276180:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.276183:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.276187:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276194:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.276201:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.276203:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.276206:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.276208:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24330, 24330] 00000080:00200000:0.0:1775764896.276216:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.276218:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.276238:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99659776 i_blocks 90112 00000080:00200000:0.0:1775764896.276241:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.276252:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99659776, count: 4096 00000080:00200000:0.0:1775764896.276258:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99659776, bytes: 4096 00000080:00200000:0.0:1775764896.276260:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.276262:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24332, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.276263:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99659776, 99663872) 0 0 00000080:00200000:0.0:1775764896.276265:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.276269:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24331, 24331] 00020000:00200000:0.0:1775764896.276270:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99659776 chunk: [99659776, 99663872) 99663872, 4096 00020000:00200000:0.0:1775764896.276272:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.276279:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24331, 24331] 00000008:00010000:0.0:1775764896.276282:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.276284:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.276289:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276295:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99659776 99663871) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276301:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276305:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276309:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.276314:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99659776, 99663872) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.276318:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24331 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.276342:0:18302:0:(rw26.c:858:ll_write_end()) pos 99659776, len 4096, copied 4096 00000080:00200000:0.0:1775764896.276345:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001859d477[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.276347:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004c00b24c 17ffffc0001001 3:0 ffff90c21282eb60 24331 lru 00000080:00200000:0.0:1775764896.276351:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000748d5811 24331: 1< 0 - - > 2< 99659776 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 954368 - | + - + - > 5< + - + - | 0 - | 3851 - -> 00000080:00200000:0.0:1775764896.276352:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001859d477 00000080:00200000:0.0:1775764896.276353:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.276354:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.276359:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.276364:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.276369:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.276373:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.276380:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.276381:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.276382:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.276389:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.276391:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.276392:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.276395:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276400:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.276405:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.276407:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.276410:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.276411:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24331, 24331] 00000080:00200000:0.0:1775764896.276419:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.276421:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.276448:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99663872 i_blocks 90112 00000080:00200000:0.0:1775764896.276451:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.276462:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99663872, count: 4096 00000080:00200000:0.0:1775764896.276468:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99663872, bytes: 4096 00000080:00200000:0.0:1775764896.276470:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.276472:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24333, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.276473:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99663872, 99667968) 0 0 00000080:00200000:0.0:1775764896.276475:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.276479:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24332, 24332] 00020000:00200000:0.0:1775764896.276481:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99663872 chunk: [99663872, 99667968) 99667968, 4096 00020000:00200000:0.0:1775764896.276489:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.276496:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24332, 24332] 00000008:00010000:0.0:1775764896.276499:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.276501:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.276511:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276517:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99663872 99667967) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276523:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276528:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276531:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.276540:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99663872, 99667968) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.276544:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24332 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.276556:0:18302:0:(rw26.c:858:ll_write_end()) pos 99663872, len 4096, copied 4096 00000080:00200000:0.0:1775764896.276558:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fbfbad20[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.276560:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005656ecf2 17ffffc0001001 3:0 ffff90c21282e000 24332 lru 00000080:00200000:0.0:1775764896.276564:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000aa9a444 24332: 1< 0 - - > 2< 99663872 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 950272 - | + - + - > 5< + - + - | 0 - | 3852 - -> 00000080:00200000:0.0:1775764896.276566:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fbfbad20 00000080:00200000:0.0:1775764896.276567:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.276568:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.276573:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.276578:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.276582:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.276590:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.276604:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.276606:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.276608:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.276614:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.276616:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.276617:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.276626:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.276631:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.276640:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.276642:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.276644:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.276646:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24332, 24332] 00000080:00200000:0.0:1775764896.276653:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.276655:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.277241:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99667968 i_blocks 90112 00000080:00200000:0.0:1775764896.277245:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.277261:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99667968, count: 4096 00000080:00200000:0.0:1775764896.277271:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99667968, bytes: 4096 00000080:00200000:0.0:1775764896.277273:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.277277:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24334, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.277278:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99667968, 99672064) 0 0 00000080:00200000:0.0:1775764896.277279:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.277284:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24333, 24333] 00020000:00200000:0.0:1775764896.277287:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99667968 chunk: [99667968, 99672064) 99672064, 4096 00020000:00200000:0.0:1775764896.277288:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.277297:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24333, 24333] 00000008:00010000:0.0:1775764896.277301:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.277303:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.277310:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.277317:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99667968 99672063) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.277385:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.277391:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.277396:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.277402:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99667968, 99672064) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.277407:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24333 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.277426:0:18302:0:(rw26.c:858:ll_write_end()) pos 99667968, len 4096, copied 4096 00000080:00200000:0.0:1775764896.277429:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000018f8c71f[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.277431:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006b0fefa5 17ffffc0001001 3:0 ffff90c21282eee0 24333 lru 00000080:00200000:0.0:1775764896.277558:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004ee1f484 24333: 1< 0 - - > 2< 99667968 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 946176 - | + - + - > 5< + - + - | 0 - | 3853 - -> 00000080:00200000:0.0:1775764896.277560:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000018f8c71f 00000080:00200000:0.0:1775764896.277561:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.277564:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.277573:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.277578:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.277582:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.277587:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.277602:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.277604:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.277606:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.277616:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.277623:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.277625:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.277628:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.277634:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.277639:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.277641:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.277644:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.277647:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24333, 24333] 00000080:00200000:0.0:1775764896.277656:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.277657:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.278001:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99672064 i_blocks 90112 00000080:00200000:0.0:1775764896.278006:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.278022:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99672064, count: 4096 00000080:00200000:0.0:1775764896.278035:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99672064, bytes: 4096 00000080:00200000:0.0:1775764896.278038:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.278041:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24335, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.278043:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99672064, 99676160) 0 0 00000080:00200000:0.0:1775764896.278045:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.278052:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24334, 24334] 00020000:00200000:0.0:1775764896.278055:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99672064 chunk: [99672064, 99676160) 99676160, 4096 00020000:00200000:0.0:1775764896.278058:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.278070:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24334, 24334] 00000008:00010000:0.0:1775764896.278076:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.278079:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.278088:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278097:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99672064 99676159) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278105:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278124:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278137:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.278145:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99672064, 99676160) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.278149:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24334 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.278168:0:18302:0:(rw26.c:858:ll_write_end()) pos 99672064, len 4096, copied 4096 00000080:00200000:0.0:1775764896.278172:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b8f5601b[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.278175:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003c53ec87 17ffffc0001001 3:0 ffff90c21282e0e0 24334 lru 00000080:00200000:0.0:1775764896.278180:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f258dd73 24334: 1< 0 - - > 2< 99672064 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 942080 - | + - + - > 5< + - + - | 0 - | 3854 - -> 00000080:00200000:0.0:1775764896.278182:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b8f5601b 00000080:00200000:0.0:1775764896.278183:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.278186:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.278194:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.278202:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.278210:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.278218:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.278230:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.278232:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.278234:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.278243:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.278245:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.278248:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.278252:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278260:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.278269:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.278272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.278277:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.278280:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24334, 24334] 00000080:00200000:0.0:1775764896.278291:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.278294:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.278338:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99676160 i_blocks 90112 00000080:00200000:0.0:1775764896.278342:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.278360:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99676160, count: 4096 00000080:00200000:0.0:1775764896.278372:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99676160, bytes: 4096 00000080:00200000:0.0:1775764896.278375:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.278379:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24336, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.278380:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99676160, 99680256) 0 0 00000080:00200000:0.0:1775764896.278383:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.278388:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24335, 24335] 00020000:00200000:0.0:1775764896.278392:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99676160 chunk: [99676160, 99680256) 99680256, 4096 00020000:00200000:0.0:1775764896.278394:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.278406:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24335, 24335] 00000008:00010000:0.0:1775764896.278412:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.278414:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.278422:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278431:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99676160 99680255) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278453:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278461:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278468:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.278475:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99676160, 99680256) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.278488:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24335 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.278514:0:18302:0:(rw26.c:858:ll_write_end()) pos 99676160, len 4096, copied 4096 00000080:00200000:0.0:1775764896.278519:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008f95f177[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.278521:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000693a0d5f 17ffffc0001001 3:0 ffff90c21282e1c0 24335 lru 00000080:00200000:0.0:1775764896.278527:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000de8dbbeb 24335: 1< 0 - - > 2< 99676160 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 937984 - | + - + - > 5< + - + - | 0 - | 3855 - -> 00000080:00200000:0.0:1775764896.278529:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008f95f177 00000080:00200000:0.0:1775764896.278530:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.278532:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.278547:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.278555:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.278562:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.278570:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.278581:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.278584:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.278586:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.278606:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.278609:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.278612:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.278616:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.278632:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.278641:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.278643:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.278648:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.278651:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24335, 24335] 00000080:00200000:0.0:1775764896.278662:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.278665:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.279384:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99680256 i_blocks 90112 00000080:00200000:0.0:1775764896.279391:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.279413:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99680256, count: 4096 00000080:00200000:0.0:1775764896.279427:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99680256, bytes: 4096 00000080:00200000:0.0:1775764896.279430:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.279445:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24337, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.279447:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99680256, 99684352) 0 0 00000080:00200000:0.0:1775764896.279450:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.279459:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24336, 24336] 00020000:00200000:0.0:1775764896.279463:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99680256 chunk: [99680256, 99684352) 99684352, 4096 00020000:00200000:0.0:1775764896.279466:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.279488:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24336, 24336] 00000008:00010000:0.0:1775764896.279497:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.279500:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.279522:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.279534:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99680256 99684351) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.279553:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.279559:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.279563:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.279569:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99680256, 99684352) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.279576:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24336 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.279605:0:18302:0:(rw26.c:858:ll_write_end()) pos 99680256, len 4096, copied 4096 00000080:00200000:0.0:1775764896.279610:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000037c1808d[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.279613:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000064a41c57 17ffffc0001001 3:0 ffff90c21282e7e0 24336 lru 00000080:00200000:0.0:1775764896.279630:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009a1d58f2 24336: 1< 0 - - > 2< 99680256 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 933888 - | + - + - > 5< + - + - | 0 - | 3856 - -> 00000080:00200000:0.0:1775764896.279633:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000037c1808d 00000080:00200000:0.0:1775764896.279634:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.279637:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.279648:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.279656:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.279662:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.279667:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.279685:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.279688:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.279690:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.279704:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.279706:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.279709:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.279713:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.279722:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.279732:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.279734:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.279739:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.279742:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24336, 24336] 00000080:00200000:0.0:1775764896.279751:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.279753:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.279795:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99684352 i_blocks 90112 00000080:00200000:0.0:1775764896.279799:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.279816:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99684352, count: 4096 00000080:00200000:0.0:1775764896.279826:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99684352, bytes: 4096 00000080:00200000:0.0:1775764896.279828:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.279831:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24338, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.279832:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99684352, 99688448) 0 0 00000080:00200000:0.0:1775764896.279834:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.279838:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24337, 24337] 00020000:00200000:0.0:1775764896.279840:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99684352 chunk: [99684352, 99688448) 99688448, 4096 00020000:00200000:0.0:1775764896.279841:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.279848:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24337, 24337] 00000008:00010000:0.0:1775764896.279852:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.279854:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.279860:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.279866:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99684352 99688447) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.279895:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.279918:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.279924:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.279932:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99684352, 99688448) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.279938:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24337 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.279953:0:18302:0:(rw26.c:858:ll_write_end()) pos 99684352, len 4096, copied 4096 00000080:00200000:0.0:1775764896.279956:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003e6056e0[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.279958:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008d79de12 17ffffc0001001 3:0 ffff90c21282e9a0 24337 lru 00000080:00200000:0.0:1775764896.279962:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005d6aa1e2 24337: 1< 0 - - > 2< 99684352 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 929792 - | + - + - > 5< + - + - | 0 - | 3857 - -> 00000080:00200000:0.0:1775764896.279963:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003e6056e0 00000080:00200000:0.0:1775764896.279964:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.279966:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.279973:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.279981:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.279987:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.279995:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.280013:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.280017:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.280019:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.280030:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.280032:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.280035:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.280038:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280043:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.280049:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.280050:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.280053:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.280055:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24337, 24337] 00000080:00200000:0.0:1775764896.280063:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.280065:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.280084:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99688448 i_blocks 90112 00000080:00200000:0.0:1775764896.280087:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.280098:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99688448, count: 4096 00000080:00200000:0.0:1775764896.280104:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99688448, bytes: 4096 00000080:00200000:0.0:1775764896.280106:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.280108:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24339, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.280109:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99688448, 99692544) 0 0 00000080:00200000:0.0:1775764896.280111:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.280127:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24338, 24338] 00020000:00200000:0.0:1775764896.280131:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99688448 chunk: [99688448, 99692544) 99692544, 4096 00020000:00200000:0.0:1775764896.280134:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.280146:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24338, 24338] 00000008:00010000:0.0:1775764896.280151:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.280154:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.280161:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280170:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99688448 99692543) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280178:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280185:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280192:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.280200:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99688448, 99692544) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.280205:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24338 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.280221:0:18302:0:(rw26.c:858:ll_write_end()) pos 99688448, len 4096, copied 4096 00000080:00200000:0.0:1775764896.280225:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f492aafe[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.280227:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a4c4a5ce 17ffffc0001001 3:0 ffff90c21282e2a0 24338 lru 00000080:00200000:0.0:1775764896.280232:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000082d2842c 24338: 1< 0 - - > 2< 99688448 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 925696 - | + - + - > 5< + - + - | 0 - | 3858 - -> 00000080:00200000:0.0:1775764896.280234:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f492aafe 00000080:00200000:0.0:1775764896.280235:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.280237:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.280245:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.280252:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.280259:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.280267:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.280279:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.280282:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.280284:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.280294:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.280296:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.280298:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.280302:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280310:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.280317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.280320:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.280324:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.280327:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24338, 24338] 00000080:00200000:0.0:1775764896.280338:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.280341:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.280371:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99692544 i_blocks 90112 00000080:00200000:0.0:1775764896.280374:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.280389:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99692544, count: 4096 00000080:00200000:0.0:1775764896.280400:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99692544, bytes: 4096 00000080:00200000:0.0:1775764896.280403:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.280406:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24340, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.280408:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99692544, 99696640) 0 0 00000080:00200000:0.0:1775764896.280410:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.280414:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24339, 24339] 00020000:00200000:0.0:1775764896.280417:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99692544 chunk: [99692544, 99696640) 99696640, 4096 00020000:00200000:0.0:1775764896.280418:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.280426:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24339, 24339] 00000008:00010000:0.0:1775764896.280430:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.280431:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.280439:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280447:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99692544 99696639) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280456:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280463:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280471:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.280480:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99692544, 99696640) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.280485:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24339 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.280501:0:18302:0:(rw26.c:858:ll_write_end()) pos 99692544, len 4096, copied 4096 00000080:00200000:0.0:1775764896.280505:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001757cbf5[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.280508:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006437b9c7 17ffffc0001001 3:0 ffff90c21282ed20 24339 lru 00000080:00200000:0.0:1775764896.280513:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004c857c54 24339: 1< 0 - - > 2< 99692544 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 921600 - | + - + - > 5< + - + - | 0 - | 3859 - -> 00000080:00200000:0.0:1775764896.280515:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001757cbf5 00000080:00200000:0.0:1775764896.280516:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.280518:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.280526:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.280534:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.280541:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.280549:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.280560:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.280563:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.280565:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.280574:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.280577:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.280580:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.280585:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280593:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.280601:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.280604:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.280608:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.280611:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24339, 24339] 00000080:00200000:0.0:1775764896.280622:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.280625:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.280654:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99696640 i_blocks 90112 00000080:00200000:0.0:1775764896.280658:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.280674:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99696640, count: 4096 00000080:00200000:0.0:1775764896.280685:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99696640, bytes: 4096 00000080:00200000:0.0:1775764896.280687:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.280691:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24341, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.280693:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99696640, 99700736) 0 0 00000080:00200000:0.0:1775764896.280695:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.280699:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24340, 24340] 00020000:00200000:0.0:1775764896.280702:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99696640 chunk: [99696640, 99700736) 99700736, 4096 00020000:00200000:0.0:1775764896.280705:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.280716:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24340, 24340] 00000008:00010000:0.0:1775764896.280721:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.280723:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.280730:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280739:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99696640 99700735) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280746:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280750:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280754:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.280761:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99696640, 99700736) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.280766:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24340 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.280793:0:18302:0:(rw26.c:858:ll_write_end()) pos 99696640, len 4096, copied 4096 00000080:00200000:0.0:1775764896.280797:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d5a4e106[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.280801:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000081496109 17ffffc0001001 3:0 ffff90c21282f2a0 24340 lru 00000080:00200000:0.0:1775764896.280807:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f0822107 24340: 1< 0 - - > 2< 99696640 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 917504 - | + - + - > 5< + - + - | 0 - | 3860 - -> 00000080:00200000:0.0:1775764896.280809:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d5a4e106 00000080:00200000:0.0:1775764896.280810:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.280812:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.280820:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.280825:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.280830:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.280838:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.280846:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.280848:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.280849:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.280856:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.280858:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.280859:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.280862:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280867:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.280892:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.280893:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.280897:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.280899:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24340, 24340] 00000080:00200000:0.0:1775764896.280906:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.280908:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.280927:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99700736 i_blocks 90112 00000080:00200000:0.0:1775764896.280930:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.280940:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99700736, count: 4096 00000080:00200000:0.0:1775764896.280946:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99700736, bytes: 4096 00000080:00200000:0.0:1775764896.280948:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.280951:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24342, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.280952:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99700736, 99704832) 0 0 00000080:00200000:0.0:1775764896.280953:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.280957:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24341, 24341] 00020000:00200000:0.0:1775764896.280959:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99700736 chunk: [99700736, 99704832) 99704832, 4096 00020000:00200000:0.0:1775764896.280961:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.280970:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24341, 24341] 00000008:00010000:0.0:1775764896.280975:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.280978:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.280985:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.280994:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99700736 99704831) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.281001:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.281009:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.281015:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.281021:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99700736, 99704832) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.281025:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24341 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.281044:0:18302:0:(rw26.c:858:ll_write_end()) pos 99700736, len 4096, copied 4096 00000080:00200000:0.0:1775764896.281046:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000aacad24e[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.281048:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000011635c8a 17ffffc0001021 2:0 ffff90c21282fd20 24341 lru 00000080:00200000:0.0:1775764896.281052:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000674bc519 24341: 1< 0 - - > 2< 99700736 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 913408 - | + - + - > 5< + - + - | 0 - | 3861 - -> 00000080:00200000:0.0:1775764896.281054:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000aacad24e 00000080:00200000:0.0:1775764896.281054:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.281056:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.281063:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.281070:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.281077:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.281084:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.281095:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.281098:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.281100:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.281111:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.281397:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.281400:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.281405:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.281412:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.281421:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.281423:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.281429:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.281432:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24341, 24341] 00000080:00200000:0.0:1775764896.281443:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.281446:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.281482:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99704832 i_blocks 90112 00000080:00200000:0.0:1775764896.281486:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.281502:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99704832, count: 4096 00000080:00200000:0.0:1775764896.281514:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99704832, bytes: 4096 00000080:00200000:0.0:1775764896.281518:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.281521:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24343, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.281523:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99704832, 99708928) 0 0 00000080:00200000:0.0:1775764896.281525:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.281531:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24342, 24342] 00020000:00200000:0.0:1775764896.281534:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99704832 chunk: [99704832, 99708928) 99708928, 4096 00020000:00200000:0.0:1775764896.281537:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.281549:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24342, 24342] 00000008:00010000:0.0:1775764896.281555:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.281558:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.281567:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.281577:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99704832 99708927) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.281585:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.281592:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.281605:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.281613:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99704832, 99708928) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.281627:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24342 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.281647:0:18302:0:(rw26.c:858:ll_write_end()) pos 99704832, len 4096, copied 4096 00000080:00200000:0.0:1775764896.281651:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e8aafbf4[3 00000000b99c1f5d 2 1 00000000bbac367d] 00000080:00200000:0.0:1775764896.281654:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b6101262 17ffffc0001001 3:0 ffff90c21282f380 24342 lru 00000080:00200000:0.0:1775764896.281659:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005325f505 24342: 1< 0 - - > 2< 99704832 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 909312 - | + - + - > 5< + - + - | 0 - | 3862 - -> 00000080:00200000:0.0:1775764896.281661:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e8aafbf4 00000080:00200000:0.0:1775764896.281662:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.281664:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.281672:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.281680:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000003a97afe6 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.281688:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.281695:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.281706:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.281709:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.281711:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.281722:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.281724:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.281727:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.281730:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.281738:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.281746:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1128 (ffff90c210c81758/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.281749:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81758/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.281753:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.281756:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24342, 24342] 00000080:00200000:0.0:1775764896.281767:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.281865:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.281973:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99708928 i_blocks 90112 00000080:00200000:1.0:1775764896.281979:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.282013:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99708928, count: 4096 00000080:00200000:0.0:1775764896.282067:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99708928, bytes: 4096 00000080:00200000:0.0:1775764896.282071:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.282076:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24344, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.282078:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99708928, 99713024) 0 0 00000080:00200000:0.0:1775764896.282080:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.282086:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24343, 24343] 00020000:00200000:0.0:1775764896.282090:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99708928 chunk: [99708928, 99713024) 99713024, 4096 00020000:00200000:0.0:1775764896.282092:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.282107:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24343, 24343] 00000008:00010000:0.0:1775764896.282124:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.282127:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.282311:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282326:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99708928 99713023) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282335:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282342:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282349:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.282356:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99708928, 99713024) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.282361:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24343 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.282381:0:18302:0:(rw26.c:858:ll_write_end()) pos 99708928, len 4096, copied 4096 00000080:00200000:0.0:1775764896.282385:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007fe932c8[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.282388:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000026b82fc9 17ffffc0001001 3:0 ffff90c21282f620 24343 lru 00000080:00200000:0.0:1775764896.282394:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008f3fb703 24343: 1< 0 - - > 2< 99708928 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 905216 - | + - + - > 5< + - + - | 0 - | 3863 - -> 00000080:00200000:0.0:1775764896.282396:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007fe932c8 00000080:00200000:0.0:1775764896.282397:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.282399:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.282408:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.282416:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.282428:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.282436:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.282448:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.282451:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.282453:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.282465:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c2195eeb48/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.282468:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c2195eeb48/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.282470:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.282474:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282482:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.282490:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c2195eeb48/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.282492:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c2195eeb48/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.282497:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.282500:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24343, 24343] 00000080:00200000:0.0:1775764896.282511:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.282514:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.282544:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99713024 i_blocks 90112 00000080:00200000:0.0:1775764896.282547:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.282565:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99713024, count: 4096 00000080:00200000:0.0:1775764896.282575:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99713024, bytes: 4096 00000080:00200000:0.0:1775764896.282579:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.282582:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24345, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.282584:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99713024, 99717120) 0 0 00000080:00200000:0.0:1775764896.282586:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.282592:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24344, 24344] 00020000:00200000:0.0:1775764896.282595:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99713024 chunk: [99713024, 99717120) 99717120, 4096 00020000:00200000:0.0:1775764896.282597:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.282610:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24344, 24344] 00000008:00010000:0.0:1775764896.282615:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.282618:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.282625:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282634:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99713024 99717119) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282643:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282650:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282657:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.282665:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99713024, 99717120) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.282670:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24344 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.282686:0:18302:0:(rw26.c:858:ll_write_end()) pos 99713024, len 4096, copied 4096 00000080:00200000:0.0:1775764896.282690:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000045d8353a[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.282693:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002a12dad4 17ffffc0001001 3:0 ffff90c21282fe00 24344 lru 00000080:00200000:0.0:1775764896.282698:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000bc673806 24344: 1< 0 - - > 2< 99713024 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 901120 - | + - + - > 5< + - + - | 0 - | 3864 - -> 00000080:00200000:0.0:1775764896.282701:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000045d8353a 00000080:00200000:0.0:1775764896.282702:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.282704:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.282711:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.282719:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.282727:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.282734:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.282745:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.282747:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.282750:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.282759:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.282762:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.282764:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.282768:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.282894:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.282912:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.282915:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.282920:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.282923:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24344, 24344] 00000080:00200000:0.0:1775764896.282935:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.282939:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.282970:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99717120 i_blocks 90112 00000080:00200000:0.0:1775764896.282974:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.282990:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99717120, count: 4096 00000080:00200000:0.0:1775764896.283000:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99717120, bytes: 4096 00000080:00200000:0.0:1775764896.283003:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.283006:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24346, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.283008:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99717120, 99721216) 0 0 00000080:00200000:0.0:1775764896.283011:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.283016:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24345, 24345] 00020000:00200000:0.0:1775764896.283019:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99717120 chunk: [99717120, 99721216) 99721216, 4096 00020000:00200000:0.0:1775764896.283021:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.283033:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24345, 24345] 00000008:00010000:0.0:1775764896.283039:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.283042:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.283050:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283060:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99717120 99721215) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283068:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283081:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283088:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.283097:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99717120, 99721216) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.283102:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24345 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.283133:0:18302:0:(rw26.c:858:ll_write_end()) pos 99717120, len 4096, copied 4096 00000080:00200000:0.0:1775764896.283138:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000034f8c4d[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.283141:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000117a7075 17ffffc0001001 3:0 ffff90c21282f540 24345 lru 00000080:00200000:0.0:1775764896.283146:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000041dc13e1 24345: 1< 0 - - > 2< 99717120 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 897024 - | + - + - > 5< + - + - | 0 - | 3865 - -> 00000080:00200000:0.0:1775764896.283148:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000034f8c4d 00000080:00200000:0.0:1775764896.283149:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.283152:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.283160:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.283168:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.283175:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.283183:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.283201:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.283204:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.283207:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.283217:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.283220:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.283223:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.283227:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283236:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.283245:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.283248:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.283252:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.283254:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24345, 24345] 00000080:00200000:0.0:1775764896.283265:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.283268:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.283301:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99721216 i_blocks 90112 00000080:00200000:0.0:1775764896.283305:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.283321:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99721216, count: 4096 00000080:00200000:0.0:1775764896.283332:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99721216, bytes: 4096 00000080:00200000:0.0:1775764896.283334:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.283338:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24347, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.283340:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99721216, 99725312) 0 0 00000080:00200000:0.0:1775764896.283342:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.283347:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24346, 24346] 00020000:00200000:0.0:1775764896.283350:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99721216 chunk: [99721216, 99725312) 99725312, 4096 00020000:00200000:0.0:1775764896.283353:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.283364:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24346, 24346] 00000008:00010000:0.0:1775764896.283370:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.283372:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.283381:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283390:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99721216 99725311) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283399:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283406:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283414:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.283423:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99721216, 99725312) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.283429:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24346 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.283450:0:18302:0:(rw26.c:858:ll_write_end()) pos 99721216, len 4096, copied 4096 00000080:00200000:0.0:1775764896.283454:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000096de8690[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.283457:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000eb0e5350 17ffffc0001001 3:0 ffff90c21282f460 24346 lru 00000080:00200000:0.0:1775764896.283463:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b16dab4c 24346: 1< 0 - - > 2< 99721216 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 892928 - | + - + - > 5< + - + - | 0 - | 3866 - -> 00000080:00200000:0.0:1775764896.283465:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000096de8690 00000080:00200000:0.0:1775764896.283466:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.283469:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.283477:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.283484:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.283491:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.283499:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.283511:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.283514:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.283516:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.283527:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.283530:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.283532:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.283537:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283546:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.283554:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.283557:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.283561:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.283564:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24346, 24346] 00000080:00200000:0.0:1775764896.283576:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.283579:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.283610:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99725312 i_blocks 90112 00000080:00200000:0.0:1775764896.283615:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.283632:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99725312, count: 4096 00000080:00200000:0.0:1775764896.283643:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99725312, bytes: 4096 00000080:00200000:0.0:1775764896.283646:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.283650:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24348, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.283651:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99725312, 99729408) 0 0 00000080:00200000:0.0:1775764896.283653:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.283659:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24347, 24347] 00020000:00200000:0.0:1775764896.283662:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99725312 chunk: [99725312, 99729408) 99729408, 4096 00020000:00200000:0.0:1775764896.283664:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.283677:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24347, 24347] 00000008:00010000:0.0:1775764896.283683:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.283686:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.283693:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283702:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99725312 99729407) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283711:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283719:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283727:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.283736:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99725312, 99729408) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.283742:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24347 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.283759:0:18302:0:(rw26.c:858:ll_write_end()) pos 99725312, len 4096, copied 4096 00000080:00200000:0.0:1775764896.283764:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006981fede[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.283766:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fa9982ec 17ffffc0001001 3:0 ffff90c21282fa80 24347 lru 00000080:00200000:0.0:1775764896.283902:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009d54e99b 24347: 1< 0 - - > 2< 99725312 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 888832 - | + - + - > 5< + - + - | 0 - | 3867 - -> 00000080:00200000:0.0:1775764896.283905:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006981fede 00000080:00200000:0.0:1775764896.283907:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.283909:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.283919:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.283928:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.283936:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.283944:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.283956:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.283959:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.283960:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.283970:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.283973:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.283976:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.283980:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.283988:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.283996:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.283999:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.284003:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.284006:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24347, 24347] 00000080:00200000:0.0:1775764896.284019:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.284022:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.284052:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99729408 i_blocks 90112 00000080:00200000:0.0:1775764896.284056:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.284071:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99729408, count: 4096 00000080:00200000:0.0:1775764896.284082:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99729408, bytes: 4096 00000080:00200000:0.0:1775764896.284084:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.284088:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24349, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.284090:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99729408, 99733504) 0 0 00000080:00200000:0.0:1775764896.284092:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.284098:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24348, 24348] 00020000:00200000:0.0:1775764896.284101:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99729408 chunk: [99729408, 99733504) 99733504, 4096 00020000:00200000:0.0:1775764896.284103:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.284125:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24348, 24348] 00000008:00010000:0.0:1775764896.284131:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.284134:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.284142:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284151:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99729408 99733503) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284159:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284167:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284174:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.284182:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99729408, 99733504) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.284188:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24348 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.284205:0:18302:0:(rw26.c:858:ll_write_end()) pos 99729408, len 4096, copied 4096 00000080:00200000:0.0:1775764896.284209:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f07aab73[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.284212:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000000bb5a42 17ffffc0001001 3:0 ffff90c21282f700 24348 lru 00000080:00200000:0.0:1775764896.284217:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cc42cd1e 24348: 1< 0 - - > 2< 99729408 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 884736 - | + - + - > 5< + - + - | 0 - | 3868 - -> 00000080:00200000:0.0:1775764896.284219:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f07aab73 00000080:00200000:0.0:1775764896.284220:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.284222:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.284230:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.284238:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.284244:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.284258:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.284270:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.284273:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.284276:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.284287:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.284290:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.284292:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.284296:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284304:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.284313:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.284315:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.284319:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.284322:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24348, 24348] 00000080:00200000:0.0:1775764896.284332:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.284335:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.284362:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99733504 i_blocks 90112 00000080:00200000:0.0:1775764896.284366:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.284379:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99733504, count: 4096 00000080:00200000:0.0:1775764896.284389:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99733504, bytes: 4096 00000080:00200000:0.0:1775764896.284392:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.284396:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24350, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.284397:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99733504, 99737600) 0 0 00000080:00200000:0.0:1775764896.284399:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.284405:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24349, 24349] 00020000:00200000:0.0:1775764896.284408:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99733504 chunk: [99733504, 99737600) 99737600, 4096 00020000:00200000:0.0:1775764896.284410:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.284421:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24349, 24349] 00000008:00010000:0.0:1775764896.284427:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.284429:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.284437:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284445:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99733504 99737599) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284452:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284459:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284466:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.284474:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99733504, 99737600) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.284479:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24349 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.284496:0:18302:0:(rw26.c:858:ll_write_end()) pos 99733504, len 4096, copied 4096 00000080:00200000:0.0:1775764896.284500:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004cea8272[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.284503:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c61b1cd2 17ffffc0001001 3:0 ffff90c21282fc40 24349 lru 00000080:00200000:0.0:1775764896.284508:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003b002c8d 24349: 1< 0 - - > 2< 99733504 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 880640 - | + - + - > 5< + - + - | 0 - | 3869 - -> 00000080:00200000:0.0:1775764896.284510:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004cea8272 00000080:00200000:0.0:1775764896.284511:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.284513:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.284521:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.284528:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.284536:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.284544:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.284555:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.284558:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.284560:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.284570:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.284573:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.284575:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.284579:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284587:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.284595:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.284598:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.284602:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.284605:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24349, 24349] 00000080:00200000:0.0:1775764896.284615:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.284619:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.284646:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99737600 i_blocks 90112 00000080:00200000:0.0:1775764896.284650:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.284666:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99737600, count: 4096 00000080:00200000:0.0:1775764896.284676:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99737600, bytes: 4096 00000080:00200000:0.0:1775764896.284679:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.284683:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24351, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.284684:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99737600, 99741696) 0 0 00000080:00200000:0.0:1775764896.284686:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.284692:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24350, 24350] 00020000:00200000:0.0:1775764896.284695:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99737600 chunk: [99737600, 99741696) 99741696, 4096 00020000:00200000:0.0:1775764896.284698:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.284710:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24350, 24350] 00000008:00010000:0.0:1775764896.284715:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.284717:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.284725:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284734:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99737600 99741695) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284741:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284748:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284758:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.284766:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99737600, 99741696) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.284866:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24350 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.284906:0:18302:0:(rw26.c:858:ll_write_end()) pos 99737600, len 4096, copied 4096 00000080:00200000:0.0:1775764896.284910:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000068a918b6[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.284913:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000129b3bb8 17ffffc0001001 3:0 ffff90c21282f8c0 24350 lru 00000080:00200000:0.0:1775764896.284919:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a7a07621 24350: 1< 0 - - > 2< 99737600 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 876544 - | + - + - > 5< + - + - | 0 - | 3870 - -> 00000080:00200000:0.0:1775764896.284921:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000068a918b6 00000080:00200000:0.0:1775764896.284923:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.284925:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.284932:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.284940:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.284947:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.284954:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.284966:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.284969:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.284972:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.284982:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.284985:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.284987:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.284991:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.284998:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.285006:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.285009:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.285013:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.285017:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24350, 24350] 00000080:00200000:0.0:1775764896.285027:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.285030:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.285061:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99741696 i_blocks 90112 00000080:00200000:0.0:1775764896.285064:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.285081:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99741696, count: 4096 00000080:00200000:0.0:1775764896.285092:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99741696, bytes: 4096 00000080:00200000:0.0:1775764896.285094:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.285098:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24352, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.285100:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99741696, 99745792) 0 0 00000080:00200000:0.0:1775764896.285102:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.285107:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24351, 24351] 00020000:00200000:0.0:1775764896.285110:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99741696 chunk: [99741696, 99745792) 99745792, 4096 00020000:00200000:0.0:1775764896.285123:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.285136:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24351, 24351] 00000008:00010000:0.0:1775764896.285142:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.285145:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.285153:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285162:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99741696 99745791) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285170:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285177:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285183:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.285191:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99741696, 99745792) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.285197:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24351 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.285213:0:18302:0:(rw26.c:858:ll_write_end()) pos 99741696, len 4096, copied 4096 00000080:00200000:0.0:1775764896.285218:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001ad016b4[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.285221:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000042a7693e 17ffffc0001001 3:0 ffff90c21282fb60 24351 lru 00000080:00200000:0.0:1775764896.285225:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a73a1f4d 24351: 1< 0 - - > 2< 99741696 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 872448 - | + - + - > 5< + - + - | 0 - | 3871 - -> 00000080:00200000:0.0:1775764896.285227:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001ad016b4 00000080:00200000:0.0:1775764896.285228:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.285230:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.285238:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.285245:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.285253:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.285261:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.285272:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.285275:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.285277:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.285287:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.285289:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.285292:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.285296:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285303:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.285311:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.285313:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.285317:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.285320:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24351, 24351] 00000080:00200000:0.0:1775764896.285330:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.285333:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.285362:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99745792 i_blocks 90112 00000080:00200000:0.0:1775764896.285366:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.285381:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99745792, count: 4096 00000080:00200000:0.0:1775764896.285392:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99745792, bytes: 4096 00000080:00200000:0.0:1775764896.285395:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.285399:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24353, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.285401:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99745792, 99749888) 0 0 00000080:00200000:0.0:1775764896.285403:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.285409:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24352, 24352] 00020000:00200000:0.0:1775764896.285412:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99745792 chunk: [99745792, 99749888) 99749888, 4096 00020000:00200000:0.0:1775764896.285414:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.285426:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24352, 24352] 00000008:00010000:0.0:1775764896.285431:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.285434:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.285442:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285450:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99745792 99749887) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285459:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285466:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285473:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.285481:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99745792, 99749888) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.285486:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24352 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.285505:0:18302:0:(rw26.c:858:ll_write_end()) pos 99745792, len 4096, copied 4096 00000080:00200000:0.0:1775764896.285509:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001f0afb34[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.285512:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fe0a5070 17ffffc0001001 3:0 ffff90c21282f000 24352 lru 00000080:00200000:0.0:1775764896.285517:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a9ba146c 24352: 1< 0 - - > 2< 99745792 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 868352 - | + - + - > 5< + - + - | 0 - | 3872 - -> 00000080:00200000:0.0:1775764896.285519:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001f0afb34 00000080:00200000:0.0:1775764896.285520:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.285525:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.285533:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.285540:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.285548:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.285555:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.285566:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.285569:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.285571:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.285581:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.285584:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.285586:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.285590:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.285598:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.285608:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.285610:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.285615:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.285618:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24352, 24352] 00000080:00200000:0.0:1775764896.285628:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.285631:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.285917:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99749888 i_blocks 90112 00000080:00200000:0.0:1775764896.285922:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.285942:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99749888, count: 4096 00000080:00200000:0.0:1775764896.285955:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99749888, bytes: 4096 00000080:00200000:0.0:1775764896.285958:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.285963:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24354, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.285964:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99749888, 99753984) 0 0 00000080:00200000:0.0:1775764896.285967:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.285973:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24353, 24353] 00020000:00200000:0.0:1775764896.285977:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99749888 chunk: [99749888, 99753984) 99753984, 4096 00020000:00200000:0.0:1775764896.285979:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.285992:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24353, 24353] 00000008:00010000:0.0:1775764896.285999:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.286002:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.286010:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286020:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99749888 99753983) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286029:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286042:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286049:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.286057:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99749888, 99753984) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.286063:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24353 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.286079:0:18302:0:(rw26.c:858:ll_write_end()) pos 99749888, len 4096, copied 4096 00000080:00200000:0.0:1775764896.286083:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c1c5e055[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.286085:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000afcc5824 17ffffc0001001 3:0 ffff90c21282fee0 24353 lru 00000080:00200000:0.0:1775764896.286091:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ccfc1d6c 24353: 1< 0 - - > 2< 99749888 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 864256 - | + - + - > 5< + - + - | 0 - | 3873 - -> 00000080:00200000:0.0:1775764896.286093:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c1c5e055 00000080:00200000:0.0:1775764896.286094:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.286096:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.286105:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.286124:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.286132:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.286140:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.286157:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.286160:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.286163:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.286174:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.286176:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.286178:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.286182:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286189:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.286198:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.286200:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.286204:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.286207:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24353, 24353] 00000080:00200000:0.0:1775764896.286219:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.286222:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.286254:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99753984 i_blocks 90112 00000080:00200000:0.0:1775764896.286257:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.286273:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99753984, count: 4096 00000080:00200000:0.0:1775764896.286284:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99753984, bytes: 4096 00000080:00200000:0.0:1775764896.286286:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.286290:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24355, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.286292:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99753984, 99758080) 0 0 00000080:00200000:0.0:1775764896.286294:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.286299:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24354, 24354] 00020000:00200000:0.0:1775764896.286303:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99753984 chunk: [99753984, 99758080) 99758080, 4096 00020000:00200000:0.0:1775764896.286305:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.286318:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24354, 24354] 00000008:00010000:0.0:1775764896.286323:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.286326:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.286335:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286344:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99753984 99758079) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286353:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286361:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286368:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.286376:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99753984, 99758080) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.286381:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24354 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.286397:0:18302:0:(rw26.c:858:ll_write_end()) pos 99753984, len 4096, copied 4096 00000080:00200000:0.0:1775764896.286401:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002795ed4c[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.286404:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000082c56529 17ffffc0001001 3:0 ffff90c21282f0e0 24354 lru 00000080:00200000:0.0:1775764896.286409:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001fb79b33 24354: 1< 0 - - > 2< 99753984 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 860160 - | + - + - > 5< + - + - | 0 - | 3874 - -> 00000080:00200000:0.0:1775764896.286411:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002795ed4c 00000080:00200000:0.0:1775764896.286412:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.286414:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.286423:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.286431:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.286438:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.286446:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.286458:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.286462:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.286464:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.286475:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.286479:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.286481:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.286486:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286493:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.286501:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.286504:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.286508:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.286511:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24354, 24354] 00000080:00200000:0.0:1775764896.286522:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.286525:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.286555:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99758080 i_blocks 90112 00000080:00200000:0.0:1775764896.286559:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.286574:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99758080, count: 4096 00000080:00200000:0.0:1775764896.286585:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99758080, bytes: 4096 00000080:00200000:0.0:1775764896.286588:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.286592:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24356, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.286594:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99758080, 99762176) 0 0 00000080:00200000:0.0:1775764896.286596:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.286602:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24355, 24355] 00020000:00200000:0.0:1775764896.286604:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99758080 chunk: [99758080, 99762176) 99762176, 4096 00020000:00200000:0.0:1775764896.286607:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.286618:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24355, 24355] 00000008:00010000:0.0:1775764896.286623:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.286626:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.286634:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286643:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99758080 99762175) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286651:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286658:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286665:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.286674:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99758080, 99762176) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.286679:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24355 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.286699:0:18302:0:(rw26.c:858:ll_write_end()) pos 99758080, len 4096, copied 4096 00000080:00200000:0.0:1775764896.286703:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000054efec40[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.286706:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000047c3c76a 17ffffc0001001 3:0 ffff90c21282f1c0 24355 lru 00000080:00200000:0.0:1775764896.286711:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000046718e4d 24355: 1< 0 - - > 2< 99758080 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 856064 - | + - + - > 5< + - + - | 0 - | 3875 - -> 00000080:00200000:0.0:1775764896.286713:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000054efec40 00000080:00200000:0.0:1775764896.286714:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.286717:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.286724:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.286733:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.286740:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.286747:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.286758:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.286761:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.286763:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.286785:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.286788:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.286791:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.286796:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286803:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.286811:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.286814:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.286819:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.286822:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24355, 24355] 00000080:00200000:0.0:1775764896.286833:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.286836:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.286866:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99762176 i_blocks 90112 00000080:00200000:0.0:1775764896.286892:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.286910:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99762176, count: 4096 00000080:00200000:0.0:1775764896.286922:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99762176, bytes: 4096 00000080:00200000:0.0:1775764896.286925:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.286929:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24357, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.286931:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99762176, 99766272) 0 0 00000080:00200000:0.0:1775764896.286933:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.286939:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24356, 24356] 00020000:00200000:0.0:1775764896.286942:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99762176 chunk: [99762176, 99766272) 99766272, 4096 00020000:00200000:0.0:1775764896.286945:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.286957:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24356, 24356] 00000008:00010000:0.0:1775764896.286963:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.286966:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.286974:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286984:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99762176 99766271) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.286992:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287000:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287008:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.287015:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99762176, 99766272) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.287021:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24356 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.287042:0:18302:0:(rw26.c:858:ll_write_end()) pos 99762176, len 4096, copied 4096 00000080:00200000:0.0:1775764896.287047:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000791ff211[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.287050:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009b14ddaf 17ffffc0001021 2:0 ffff90c21282f7e0 24356 lru 00000080:00200000:0.0:1775764896.287055:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000099fa3c65 24356: 1< 0 - - > 2< 99762176 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 851968 - | + - + - > 5< + - + - | 0 - | 3876 - -> 00000080:00200000:0.0:1775764896.287057:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000791ff211 00000080:00200000:0.0:1775764896.287058:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.287060:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.287068:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.287076:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.287083:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.287096:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.287109:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.287124:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.287127:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.287138:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.287141:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.287144:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.287148:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287156:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.287165:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.287167:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.287171:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.287174:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24356, 24356] 00000080:00200000:0.0:1775764896.287185:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.287188:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.287218:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99766272 i_blocks 90112 00000080:00200000:0.0:1775764896.287222:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.287240:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99766272, count: 4096 00000080:00200000:0.0:1775764896.287251:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99766272, bytes: 4096 00000080:00200000:0.0:1775764896.287254:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.287258:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24358, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.287259:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99766272, 99770368) 0 0 00000080:00200000:0.0:1775764896.287261:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.287266:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24357, 24357] 00020000:00200000:0.0:1775764896.287269:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99766272 chunk: [99766272, 99770368) 99770368, 4096 00020000:00200000:0.0:1775764896.287272:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.287284:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24357, 24357] 00000008:00010000:0.0:1775764896.287289:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.287292:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.287299:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287308:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99766272 99770367) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287316:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287323:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287330:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.287338:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99766272, 99770368) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.287344:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24357 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.287361:0:18302:0:(rw26.c:858:ll_write_end()) pos 99766272, len 4096, copied 4096 00000080:00200000:0.0:1775764896.287366:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004ee7bf66[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.287369:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000426ffc89 17ffffc0001001 3:0 ffff90c21282f9a0 24357 lru 00000080:00200000:0.0:1775764896.287374:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005e207831 24357: 1< 0 - - > 2< 99766272 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 847872 - | + - + - > 5< + - + - | 0 - | 3877 - -> 00000080:00200000:0.0:1775764896.287377:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004ee7bf66 00000080:00200000:0.0:1775764896.287378:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.287380:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.287389:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.287397:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.287404:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.287411:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.287423:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.287426:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.287428:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.287438:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.287441:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.287444:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.287448:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287456:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.287465:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.287468:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.287473:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.287476:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24357, 24357] 00000080:00200000:0.0:1775764896.287488:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.287491:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.287521:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99770368 i_blocks 90112 00000080:00200000:0.0:1775764896.287525:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.287541:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99770368, count: 4096 00000080:00200000:0.0:1775764896.287552:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99770368, bytes: 4096 00000080:00200000:0.0:1775764896.287555:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.287558:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24359, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.287560:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99770368, 99774464) 0 0 00000080:00200000:0.0:1775764896.287562:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.287567:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24358, 24358] 00020000:00200000:0.0:1775764896.287570:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99770368 chunk: [99770368, 99774464) 99774464, 4096 00020000:00200000:0.0:1775764896.287572:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.287583:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24358, 24358] 00000008:00010000:0.0:1775764896.287588:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.287591:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.287599:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287608:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99770368 99774463) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287616:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287624:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287636:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.287645:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99770368, 99774464) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.287651:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24358 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.287680:0:18302:0:(rw26.c:858:ll_write_end()) pos 99770368, len 4096, copied 4096 00000080:00200000:0.0:1775764896.287685:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006f3dcb0a[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.287687:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f6417c8d 17ffffc0001001 3:0 ffff90c212830620 24358 lru 00000080:00200000:0.0:1775764896.287693:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a647cda8 24358: 1< 0 - - > 2< 99770368 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 843776 - | + - + - > 5< + - + - | 0 - | 3878 - -> 00000080:00200000:0.0:1775764896.287695:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006f3dcb0a 00000080:00200000:0.0:1775764896.287695:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.287698:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.287706:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.287714:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.287721:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.287729:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.287741:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.287744:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.287747:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.287757:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.287759:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.287762:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.287766:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287775:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.287783:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.287785:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.287790:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.287793:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24358, 24358] 00000080:00200000:0.0:1775764896.287804:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.287807:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.287835:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99774464 i_blocks 90112 00000080:00200000:0.0:1775764896.287838:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.287853:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99774464, count: 4096 00000080:00200000:0.0:1775764896.287864:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99774464, bytes: 4096 00000080:00200000:0.0:1775764896.287866:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.287899:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24360, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.287901:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99774464, 99778560) 0 0 00000080:00200000:0.0:1775764896.287903:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.287909:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24359, 24359] 00020000:00200000:0.0:1775764896.287912:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99774464 chunk: [99774464, 99778560) 99778560, 4096 00020000:00200000:0.0:1775764896.287915:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.287929:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24359, 24359] 00000008:00010000:0.0:1775764896.287935:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.287938:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.287946:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287955:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99774464 99778559) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287964:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287971:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.287978:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.287986:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99774464, 99778560) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.287991:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24359 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.288006:0:18302:0:(rw26.c:858:ll_write_end()) pos 99774464, len 4096, copied 4096 00000080:00200000:0.0:1775764896.288010:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000c94a8cb[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.288013:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000026edd04e 17ffffc0001001 3:0 ffff90c212830e00 24359 lru 00000080:00200000:0.0:1775764896.288019:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cfe260fa 24359: 1< 0 - - > 2< 99774464 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 839680 - | + - + - > 5< + - + - | 0 - | 3879 - -> 00000080:00200000:0.0:1775764896.288021:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000c94a8cb 00000080:00200000:0.0:1775764896.288022:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.288024:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.288031:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.288039:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.288046:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.288054:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.288065:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.288067:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.288069:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.288079:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.288081:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.288084:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.288088:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288096:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.288103:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.288106:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.288110:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.288293:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24359, 24359] 00000080:00200000:0.0:1775764896.288307:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.288310:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.288342:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99778560 i_blocks 90112 00000080:00200000:0.0:1775764896.288345:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.288362:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99778560, count: 4096 00000080:00200000:0.0:1775764896.288373:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99778560, bytes: 4096 00000080:00200000:0.0:1775764896.288375:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.288379:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24361, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.288381:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99778560, 99782656) 0 0 00000080:00200000:0.0:1775764896.288383:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.288388:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24360, 24360] 00020000:00200000:0.0:1775764896.288391:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99778560 chunk: [99778560, 99782656) 99782656, 4096 00020000:00200000:0.0:1775764896.288393:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.288405:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24360, 24360] 00000008:00010000:0.0:1775764896.288411:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.288413:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.288421:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288431:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99778560 99782655) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288439:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288447:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288454:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.288462:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99778560, 99782656) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.288468:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24360 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.288485:0:18302:0:(rw26.c:858:ll_write_end()) pos 99778560, len 4096, copied 4096 00000080:00200000:0.0:1775764896.288490:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008ae5e9a1[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.288493:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a640e002 17ffffc0001001 3:0 ffff90c212830540 24360 lru 00000080:00200000:0.0:1775764896.288498:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b53b2920 24360: 1< 0 - - > 2< 99778560 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 835584 - | + - + - > 5< + - + - | 0 - | 3880 - -> 00000080:00200000:0.0:1775764896.288500:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008ae5e9a1 00000080:00200000:0.0:1775764896.288502:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.288504:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.288512:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.288520:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.288527:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.288535:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.288546:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.288549:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.288551:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.288562:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.288564:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.288567:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.288571:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288578:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.288587:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.288590:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.288594:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.288597:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24360, 24360] 00000080:00200000:0.0:1775764896.288608:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.288611:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.288641:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99782656 i_blocks 90112 00000080:00200000:0.0:1775764896.288645:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.288662:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99782656, count: 4096 00000080:00200000:0.0:1775764896.288674:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99782656, bytes: 4096 00000080:00200000:0.0:1775764896.288677:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.288680:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24362, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.288682:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99782656, 99786752) 0 0 00000080:00200000:0.0:1775764896.288684:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.288689:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24361, 24361] 00020000:00200000:0.0:1775764896.288692:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99782656 chunk: [99782656, 99786752) 99786752, 4096 00020000:00200000:0.0:1775764896.288694:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.288706:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24361, 24361] 00000008:00010000:0.0:1775764896.288711:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.288714:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.288722:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288731:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99782656 99786751) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288740:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288753:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.288760:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.288768:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99782656, 99786752) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.288862:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24361 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.288901:0:18302:0:(rw26.c:858:ll_write_end()) pos 99782656, len 4096, copied 4096 00000080:00200000:0.0:1775764896.288906:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000030f31122[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.288909:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b4b6ad0f 17ffffc0001001 3:0 ffff90c212830460 24361 lru 00000080:00200000:0.0:1775764896.288914:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ed716d76 24361: 1< 0 - - > 2< 99782656 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 831488 - | + - + - > 5< + - + - | 0 - | 3881 - -> 00000080:00200000:0.0:1775764896.288916:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000030f31122 00000080:00200000:0.0:1775764896.288917:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.288920:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.288928:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.288937:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.288945:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.288953:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.288970:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.288973:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.288975:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.288985:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.288988:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.288990:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.288994:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289003:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.289012:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.289014:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.289019:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.289022:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24361, 24361] 00000080:00200000:0.0:1775764896.289032:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.289035:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.289067:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99786752 i_blocks 90112 00000080:00200000:0.0:1775764896.289071:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.289088:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99786752, count: 4096 00000080:00200000:0.0:1775764896.289100:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99786752, bytes: 4096 00000080:00200000:0.0:1775764896.289103:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.289108:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24363, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.289109:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99786752, 99790848) 0 0 00000080:00200000:0.0:1775764896.289124:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.289131:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24362, 24362] 00020000:00200000:0.0:1775764896.289134:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99786752 chunk: [99786752, 99790848) 99790848, 4096 00020000:00200000:0.0:1775764896.289137:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.289150:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24362, 24362] 00000008:00010000:0.0:1775764896.289155:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.289157:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.289165:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289174:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99786752 99790847) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289182:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289189:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289196:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.289204:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99786752, 99790848) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.289209:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24362 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.289225:0:18302:0:(rw26.c:858:ll_write_end()) pos 99786752, len 4096, copied 4096 00000080:00200000:0.0:1775764896.289231:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007ca2803b[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.289234:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000097609463 17ffffc0001001 3:0 ffff90c212830a80 24362 lru 00000080:00200000:0.0:1775764896.289240:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004ea64053 24362: 1< 0 - - > 2< 99786752 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 827392 - | + - + - > 5< + - + - | 0 - | 3882 - -> 00000080:00200000:0.0:1775764896.289242:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007ca2803b 00000080:00200000:0.0:1775764896.289243:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.289245:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.289254:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.289262:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.289269:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.289277:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.289288:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.289291:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.289293:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.289303:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.289306:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.289308:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.289312:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289320:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.289329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.289331:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.289336:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.289338:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24362, 24362] 00000080:00200000:0.0:1775764896.289349:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.289352:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.289383:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99790848 i_blocks 90112 00000080:00200000:0.0:1775764896.289386:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.289403:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99790848, count: 4096 00000080:00200000:0.0:1775764896.289415:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99790848, bytes: 4096 00000080:00200000:0.0:1775764896.289418:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.289421:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24364, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.289423:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99790848, 99794944) 0 0 00000080:00200000:0.0:1775764896.289426:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.289430:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24363, 24363] 00020000:00200000:0.0:1775764896.289433:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99790848 chunk: [99790848, 99794944) 99794944, 4096 00020000:00200000:0.0:1775764896.289435:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.289446:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24363, 24363] 00000008:00010000:0.0:1775764896.289451:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.289453:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.289461:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289469:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99790848 99794943) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289476:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289483:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289490:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.289499:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99790848, 99794944) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.289504:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24363 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.289521:0:18302:0:(rw26.c:858:ll_write_end()) pos 99790848, len 4096, copied 4096 00000080:00200000:0.0:1775764896.289525:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004deccda0[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.289528:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a5a2c727 17ffffc0001001 3:0 ffff90c212830700 24363 lru 00000080:00200000:0.0:1775764896.289533:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f85fd3f4 24363: 1< 0 - - > 2< 99790848 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 823296 - | + - + - > 5< + - + - | 0 - | 3883 - -> 00000080:00200000:0.0:1775764896.289535:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004deccda0 00000080:00200000:0.0:1775764896.289537:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.289539:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.289546:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.289555:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.289561:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.289568:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.289579:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.289582:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.289584:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.289594:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.289597:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.289599:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.289604:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289612:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.289619:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.289622:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.289627:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.289629:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24363, 24363] 00000080:00200000:0.0:1775764896.289640:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.289643:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.289672:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99794944 i_blocks 90112 00000080:00200000:0.0:1775764896.289676:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.289692:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99794944, count: 4096 00000080:00200000:0.0:1775764896.289702:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99794944, bytes: 4096 00000080:00200000:0.0:1775764896.289705:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.289708:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24365, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.289710:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99794944, 99799040) 0 0 00000080:00200000:0.0:1775764896.289712:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.289717:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24364, 24364] 00020000:00200000:0.0:1775764896.289720:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99794944 chunk: [99794944, 99799040) 99799040, 4096 00020000:00200000:0.0:1775764896.289723:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.289734:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24364, 24364] 00000008:00010000:0.0:1775764896.289740:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.289742:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.289750:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289759:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99794944 99799039) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289768:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289851:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289856:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.289862:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99794944, 99799040) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.289866:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24364 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.289904:0:18302:0:(rw26.c:858:ll_write_end()) pos 99794944, len 4096, copied 4096 00000080:00200000:0.0:1775764896.289907:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008c907b00[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.289909:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f2be364e 17ffffc0001001 3:0 ffff90c212830c40 24364 lru 00000080:00200000:0.0:1775764896.289913:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000627f4df0 24364: 1< 0 - - > 2< 99794944 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 819200 - | + - + - > 5< + - + - | 0 - | 3884 - -> 00000080:00200000:0.0:1775764896.289914:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008c907b00 00000080:00200000:0.0:1775764896.289915:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.289917:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.289923:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.289928:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.289932:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.289941:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.289949:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.289950:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.289952:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.289959:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.289961:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.289963:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.289965:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.289970:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.289975:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.289977:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.289979:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.289981:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24364, 24364] 00000080:00200000:0.0:1775764896.289988:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.289990:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.290009:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99799040 i_blocks 90112 00000080:00200000:0.0:1775764896.290012:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.290023:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99799040, count: 4096 00000080:00200000:0.0:1775764896.290029:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99799040, bytes: 4096 00000080:00200000:0.0:1775764896.290031:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.290034:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24366, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.290035:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99799040, 99803136) 0 0 00000080:00200000:0.0:1775764896.290036:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.290040:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24365, 24365] 00020000:00200000:0.0:1775764896.290042:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99799040 chunk: [99799040, 99803136) 99803136, 4096 00020000:00200000:0.0:1775764896.290043:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.290050:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24365, 24365] 00000008:00010000:0.0:1775764896.290053:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.290055:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.290060:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290066:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99799040 99803135) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290072:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290076:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290080:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.290084:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99799040, 99803136) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.290087:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24365 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.290099:0:18302:0:(rw26.c:858:ll_write_end()) pos 99799040, len 4096, copied 4096 00000080:00200000:0.0:1775764896.290102:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ecc0d8f2[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.290104:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000564e9849 17ffffc0001001 3:0 ffff90c2128308c0 24365 lru 00000080:00200000:0.0:1775764896.290107:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009ab47520 24365: 1< 0 - - > 2< 99799040 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 815104 - | + - + - > 5< + - + - | 0 - | 3885 - -> 00000080:00200000:0.0:1775764896.290109:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ecc0d8f2 00000080:00200000:0.0:1775764896.290110:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.290111:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.290130:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.290138:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.290144:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.290151:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.290161:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.290164:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.290166:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.290176:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.290179:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.290181:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.290185:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290193:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.290201:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.290204:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.290208:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.290211:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24365, 24365] 00000080:00200000:0.0:1775764896.290222:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.290224:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.290245:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99803136 i_blocks 90112 00000080:00200000:0.0:1775764896.290249:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.290264:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99803136, count: 4096 00000080:00200000:0.0:1775764896.290275:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99803136, bytes: 4096 00000080:00200000:0.0:1775764896.290278:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.290281:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24367, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.290283:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99803136, 99807232) 0 0 00000080:00200000:0.0:1775764896.290285:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.290291:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24366, 24366] 00020000:00200000:0.0:1775764896.290293:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99803136 chunk: [99803136, 99807232) 99807232, 4096 00020000:00200000:0.0:1775764896.290296:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.290307:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24366, 24366] 00000008:00010000:0.0:1775764896.290312:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.290314:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.290321:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290327:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99803136 99807231) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290333:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290337:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290345:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.290350:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99803136, 99807232) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.290354:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24366 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.290369:0:18302:0:(rw26.c:858:ll_write_end()) pos 99803136, len 4096, copied 4096 00000080:00200000:0.0:1775764896.290371:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000070d00e44[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.290373:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e3feb360 17ffffc0001001 3:0 ffff90c212830b60 24366 lru 00000080:00200000:0.0:1775764896.290377:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004e073dcb 24366: 1< 0 - - > 2< 99803136 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 811008 - | + - + - > 5< + - + - | 0 - | 3886 - -> 00000080:00200000:0.0:1775764896.290378:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000070d00e44 00000080:00200000:0.0:1775764896.290379:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.290381:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.290386:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.290391:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.290395:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.290399:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.290406:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.290408:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.290410:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.290418:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.290421:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.290423:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.290427:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290434:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.290443:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.290445:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.290449:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.290452:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24366, 24366] 00000080:00200000:0.0:1775764896.290463:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.290466:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.290495:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99807232 i_blocks 90112 00000080:00200000:0.0:1775764896.290499:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.290514:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99807232, count: 4096 00000080:00200000:0.0:1775764896.290524:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99807232, bytes: 4096 00000080:00200000:0.0:1775764896.290527:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.290531:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24368, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.290533:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99807232, 99811328) 0 0 00000080:00200000:0.0:1775764896.290535:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.290541:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24367, 24367] 00020000:00200000:0.0:1775764896.290543:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99807232 chunk: [99807232, 99811328) 99811328, 4096 00020000:00200000:0.0:1775764896.290546:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.290557:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24367, 24367] 00000008:00010000:0.0:1775764896.290562:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.290565:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.290572:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290582:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99807232 99811327) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290590:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290598:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290605:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.290613:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99807232, 99811328) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.290618:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24367 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.290636:0:18302:0:(rw26.c:858:ll_write_end()) pos 99807232, len 4096, copied 4096 00000080:00200000:0.0:1775764896.290640:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000054b2ba06[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.290643:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a2b5b6b7 17ffffc0001001 3:0 ffff90c212830000 24367 lru 00000080:00200000:0.0:1775764896.290648:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000406fccb9 24367: 1< 0 - - > 2< 99807232 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 806912 - | + - + - > 5< + - + - | 0 - | 3887 - -> 00000080:00200000:0.0:1775764896.290650:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000054b2ba06 00000080:00200000:0.0:1775764896.290651:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.290653:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.290660:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.290668:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.290675:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.290683:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.290694:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.290697:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.290699:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.290709:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.290711:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.290714:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.290718:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.290726:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.290734:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.290736:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.290741:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.290744:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24367, 24367] 00000080:00200000:0.0:1775764896.290754:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.290758:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.290910:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99811328 i_blocks 90112 00000080:00200000:0.0:1775764896.290914:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.290931:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99811328, count: 4096 00000080:00200000:0.0:1775764896.290942:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99811328, bytes: 4096 00000080:00200000:0.0:1775764896.290945:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.290950:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24369, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.290951:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99811328, 99815424) 0 0 00000080:00200000:0.0:1775764896.290954:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.290960:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24368, 24368] 00020000:00200000:0.0:1775764896.290963:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99811328 chunk: [99811328, 99815424) 99815424, 4096 00020000:00200000:0.0:1775764896.290965:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.290977:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24368, 24368] 00000008:00010000:0.0:1775764896.290982:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.290985:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.290993:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291001:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99811328 99815423) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291011:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291019:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291026:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.291034:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99811328, 99815424) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.291039:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24368 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.291056:0:18302:0:(rw26.c:858:ll_write_end()) pos 99811328, len 4096, copied 4096 00000080:00200000:0.0:1775764896.291060:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005bc65edd[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.291063:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c10c2c26 17ffffc0001001 3:0 ffff90c212830ee0 24368 lru 00000080:00200000:0.0:1775764896.291070:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000014ce7fdd 24368: 1< 0 - - > 2< 99811328 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 802816 - | + - + - > 5< + - + - | 0 - | 3888 - -> 00000080:00200000:0.0:1775764896.291072:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005bc65edd 00000080:00200000:0.0:1775764896.291073:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.291076:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.291083:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.291091:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.291098:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.291106:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.291130:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.291133:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.291135:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.291144:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.291147:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.291150:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.291154:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291162:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.291172:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.291175:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.291179:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.291182:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24368, 24368] 00000080:00200000:0.0:1775764896.291193:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.291196:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.291224:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99815424 i_blocks 90112 00000080:00200000:0.0:1775764896.291228:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.291243:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99815424, count: 4096 00000080:00200000:0.0:1775764896.291253:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99815424, bytes: 4096 00000080:00200000:0.0:1775764896.291256:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.291260:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24370, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.291262:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99815424, 99819520) 0 0 00000080:00200000:0.0:1775764896.291264:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.291270:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24369, 24369] 00020000:00200000:0.0:1775764896.291273:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99815424 chunk: [99815424, 99819520) 99819520, 4096 00020000:00200000:0.0:1775764896.291275:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.291286:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24369, 24369] 00000008:00010000:0.0:1775764896.291291:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.291294:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.291301:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291310:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99815424 99819519) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291319:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291332:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291339:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.291347:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99815424, 99819520) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.291351:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24369 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.291367:0:18302:0:(rw26.c:858:ll_write_end()) pos 99815424, len 4096, copied 4096 00000080:00200000:0.0:1775764896.291371:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000743fb30a[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.291375:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000784152be 17ffffc0001001 3:0 ffff90c2128300e0 24369 lru 00000080:00200000:0.0:1775764896.291381:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005cc3625d 24369: 1< 0 - - > 2< 99815424 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 798720 - | + - + - > 5< + - + - | 0 - | 3889 - -> 00000080:00200000:0.0:1775764896.291383:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000743fb30a 00000080:00200000:0.0:1775764896.291384:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.291386:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.291394:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.291402:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.291409:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.291417:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.291433:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.291436:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.291439:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.291448:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.291450:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.291453:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.291457:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291466:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.291474:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.291477:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.291481:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.291484:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24369, 24369] 00000080:00200000:0.0:1775764896.291495:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.291499:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.291529:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99819520 i_blocks 90112 00000080:00200000:0.0:1775764896.291532:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.291549:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99819520, count: 4096 00000080:00200000:0.0:1775764896.291560:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99819520, bytes: 4096 00000080:00200000:0.0:1775764896.291563:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.291567:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24371, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.291569:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99819520, 99823616) 0 0 00000080:00200000:0.0:1775764896.291571:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.291576:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24370, 24370] 00020000:00200000:0.0:1775764896.291580:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99819520 chunk: [99819520, 99823616) 99823616, 4096 00020000:00200000:0.0:1775764896.291582:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.291593:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24370, 24370] 00000008:00010000:0.0:1775764896.291598:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.291600:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.291608:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291616:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99819520 99823615) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291625:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291632:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291637:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.291645:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99819520, 99823616) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.291650:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24370 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.291667:0:18302:0:(rw26.c:858:ll_write_end()) pos 99819520, len 4096, copied 4096 00000080:00200000:0.0:1775764896.291672:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000012bc3c59[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.291674:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b0e34d49 17ffffc0001001 3:0 ffff90c2128301c0 24370 lru 00000080:00200000:0.0:1775764896.291680:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006a1d618d 24370: 1< 0 - - > 2< 99819520 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 794624 - | + - + - > 5< + - + - | 0 - | 3890 - -> 00000080:00200000:0.0:1775764896.291682:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000012bc3c59 00000080:00200000:0.0:1775764896.291683:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.291685:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.291694:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.291702:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.291708:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.291715:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.291726:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.291730:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.291732:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.291742:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.291745:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.291747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.291751:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.291759:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.291767:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.291770:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.291913:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.291916:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24370, 24370] 00000080:00200000:0.0:1775764896.291927:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.291931:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.291961:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99823616 i_blocks 90112 00000080:00200000:0.0:1775764896.291965:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.291981:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99823616, count: 4096 00000080:00200000:0.0:1775764896.291992:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99823616, bytes: 4096 00000080:00200000:0.0:1775764896.291994:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.291999:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24372, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.292000:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99823616, 99827712) 0 0 00000080:00200000:0.0:1775764896.292003:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.292008:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24371, 24371] 00020000:00200000:0.0:1775764896.292011:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99823616 chunk: [99823616, 99827712) 99827712, 4096 00020000:00200000:0.0:1775764896.292014:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.292025:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24371, 24371] 00000008:00010000:0.0:1775764896.292031:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.292033:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.292042:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292052:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99823616 99827711) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292060:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292068:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292077:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.292085:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99823616, 99827712) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.292090:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24371 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.292126:0:18302:0:(rw26.c:858:ll_write_end()) pos 99823616, len 4096, copied 4096 00000080:00200000:0.0:1775764896.292131:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c0728e04[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.292134:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000dbbd5ec8 17ffffc0001021 2:0 ffff90c2128307e0 24371 lru 00000080:00200000:0.0:1775764896.292140:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000234e12bf 24371: 1< 0 - - > 2< 99823616 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 790528 - | + - + - > 5< + - + - | 0 - | 3891 - -> 00000080:00200000:0.0:1775764896.292142:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c0728e04 00000080:00200000:0.0:1775764896.292144:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.292146:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.292155:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.292162:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.292170:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.292177:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.292188:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.292191:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.292194:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.292204:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.292207:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.292210:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.292214:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292223:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.292230:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.292233:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.292237:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.292240:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24371, 24371] 00000080:00200000:0.0:1775764896.292251:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.292254:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.292281:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99827712 i_blocks 90112 00000080:00200000:0.0:1775764896.292285:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.292301:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99827712, count: 4096 00000080:00200000:0.0:1775764896.292312:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99827712, bytes: 4096 00000080:00200000:0.0:1775764896.292315:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.292319:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24373, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.292320:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99827712, 99831808) 0 0 00000080:00200000:0.0:1775764896.292323:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.292328:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24372, 24372] 00020000:00200000:0.0:1775764896.292331:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99827712 chunk: [99827712, 99831808) 99831808, 4096 00020000:00200000:0.0:1775764896.292334:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.292345:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24372, 24372] 00000008:00010000:0.0:1775764896.292350:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.292353:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.292360:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292369:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99827712 99831807) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292378:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292385:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292391:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.292399:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99827712, 99831808) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.292404:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24372 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.292421:0:18302:0:(rw26.c:858:ll_write_end()) pos 99827712, len 4096, copied 4096 00000080:00200000:0.0:1775764896.292425:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000662616ce[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.292428:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000734e422e 17ffffc0001001 3:0 ffff90c2128309a0 24372 lru 00000080:00200000:0.0:1775764896.292433:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000092714c21 24372: 1< 0 - - > 2< 99827712 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 786432 - | + - + - > 5< + - + - | 0 - | 3892 - -> 00000080:00200000:0.0:1775764896.292435:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000662616ce 00000080:00200000:0.0:1775764896.292436:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.292438:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.292445:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.292453:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.292459:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.292472:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.292483:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.292486:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.292488:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.292498:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.292501:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.292503:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.292507:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292516:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.292524:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.292526:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.292530:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.292533:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24372, 24372] 00000080:00200000:0.0:1775764896.292543:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.292547:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.292576:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99831808 i_blocks 90112 00000080:00200000:0.0:1775764896.292579:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.292596:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99831808, count: 4096 00000080:00200000:0.0:1775764896.292606:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99831808, bytes: 4096 00000080:00200000:0.0:1775764896.292609:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.292614:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24374, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.292616:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99831808, 99835904) 0 0 00000080:00200000:0.0:1775764896.292618:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.292624:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24373, 24373] 00020000:00200000:0.0:1775764896.292627:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99831808 chunk: [99831808, 99835904) 99835904, 4096 00020000:00200000:0.0:1775764896.292629:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.292641:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24373, 24373] 00000008:00010000:0.0:1775764896.292646:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.292649:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.292657:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292666:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99831808 99835903) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292675:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292682:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.292688:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.292696:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99831808, 99835904) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.292701:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24373 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.292718:0:18302:0:(rw26.c:858:ll_write_end()) pos 99831808, len 4096, copied 4096 00000080:00200000:0.0:1775764896.292722:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a028504a[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.292725:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008feac497 17ffffc0001001 3:0 ffff90c2128302a0 24373 lru 00000080:00200000:0.0:1775764896.292731:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000017cefb1e 24373: 1< 0 - - > 2< 99831808 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 782336 - | + - + - > 5< + - + - | 0 - | 3893 - -> 00000080:00200000:0.0:1775764896.292733:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a028504a 00000080:00200000:0.0:1775764896.292735:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.292737:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.292745:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.292752:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.292759:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.292765:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.292966:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.292970:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.292973:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.292983:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.292986:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.292989:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.292993:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293003:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.293011:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.293014:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.293019:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.293022:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24373, 24373] 00000080:00200000:0.0:1775764896.293033:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.293036:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.293066:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99835904 i_blocks 90112 00000080:00200000:0.0:1775764896.293070:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.293085:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99835904, count: 4096 00000080:00200000:0.0:1775764896.293096:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99835904, bytes: 4096 00000080:00200000:0.0:1775764896.293098:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.293102:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24375, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.293104:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99835904, 99840000) 0 0 00000080:00200000:0.0:1775764896.293107:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.293124:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24374, 24374] 00020000:00200000:0.0:1775764896.293127:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99835904 chunk: [99835904, 99840000) 99840000, 4096 00020000:00200000:0.0:1775764896.293130:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.293142:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24374, 24374] 00000008:00010000:0.0:1775764896.293147:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.293150:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.293159:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293168:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99835904 99839999) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293177:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293185:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293198:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.293206:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99835904, 99840000) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.293211:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24374 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.293228:0:18302:0:(rw26.c:858:ll_write_end()) pos 99835904, len 4096, copied 4096 00000080:00200000:0.0:1775764896.293232:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000668a0b71[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.293236:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005432960c 17ffffc0001001 3:0 ffff90c212830d20 24374 lru 00000080:00200000:0.0:1775764896.293242:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b1a01ba4 24374: 1< 0 - - > 2< 99835904 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 778240 - | + - + - > 5< + - + - | 0 - | 3894 - -> 00000080:00200000:0.0:1775764896.293244:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000668a0b71 00000080:00200000:0.0:1775764896.293246:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.293248:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.293255:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.293264:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.293271:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.293279:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.293290:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.293293:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.293296:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.293305:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.293308:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.293311:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.293315:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293324:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.293333:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.293335:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.293339:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.293343:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24374, 24374] 00000080:00200000:0.0:1775764896.293353:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.293356:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.293383:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99840000 i_blocks 90112 00000080:00200000:0.0:1775764896.293387:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.293402:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99840000, count: 4096 00000080:00200000:0.0:1775764896.293412:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99840000, bytes: 4096 00000080:00200000:0.0:1775764896.293415:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.293418:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24376, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.293420:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99840000, 99844096) 0 0 00000080:00200000:0.0:1775764896.293423:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.293428:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24375, 24375] 00020000:00200000:0.0:1775764896.293431:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99840000 chunk: [99840000, 99844096) 99844096, 4096 00020000:00200000:0.0:1775764896.293433:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.293444:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24375, 24375] 00000008:00010000:0.0:1775764896.293449:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.293451:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.293459:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293468:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99840000 99844095) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293477:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293483:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293490:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.293497:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99840000, 99844096) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.293502:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24375 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.293517:0:18302:0:(rw26.c:858:ll_write_end()) pos 99840000, len 4096, copied 4096 00000080:00200000:0.0:1775764896.293522:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b96e3d6d[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.293525:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001d77bff6 17ffffc0001001 3:0 ffff90c212830380 24375 lru 00000080:00200000:0.0:1775764896.293531:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000063818274 24375: 1< 0 - - > 2< 99840000 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 774144 - | + - + - > 5< + - + - | 0 - | 3895 - -> 00000080:00200000:0.0:1775764896.293533:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b96e3d6d 00000080:00200000:0.0:1775764896.293534:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.293537:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.293544:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.293552:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.293560:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.293568:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.293578:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.293581:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.293584:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.293593:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.293595:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.293598:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.293602:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293609:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.293617:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.293619:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.293623:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.293626:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24375, 24375] 00000080:00200000:0.0:1775764896.293636:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.293639:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.293665:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99844096 i_blocks 90112 00000080:00200000:0.0:1775764896.293669:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.293684:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99844096, count: 4096 00000080:00200000:0.0:1775764896.293693:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99844096, bytes: 4096 00000080:00200000:0.0:1775764896.293696:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.293700:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24377, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.293701:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99844096, 99848192) 0 0 00000080:00200000:0.0:1775764896.293704:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.293708:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24376, 24376] 00020000:00200000:0.0:1775764896.293711:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99844096 chunk: [99844096, 99848192) 99848192, 4096 00020000:00200000:0.0:1775764896.293714:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.293723:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24376, 24376] 00000008:00010000:0.0:1775764896.293728:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.293731:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.293738:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293747:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99844096 99848191) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293755:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293763:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.293769:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.293895:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99844096, 99848192) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.293901:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24376 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.293929:0:18302:0:(rw26.c:858:ll_write_end()) pos 99844096, len 4096, copied 4096 00000080:00200000:0.0:1775764896.293933:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002c2663fc[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.293936:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000088693011 17ffffc0001001 3:0 ffff90c212831380 24376 lru 00000080:00200000:0.0:1775764896.293942:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a32b0bdd 24376: 1< 0 - - > 2< 99844096 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 770048 - | + - + - > 5< + - + - | 0 - | 3896 - -> 00000080:00200000:0.0:1775764896.293944:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002c2663fc 00000080:00200000:0.0:1775764896.293945:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.293947:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.293956:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.293964:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.293971:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.293979:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.293991:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.293993:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.293995:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.294005:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.294008:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.294010:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.294014:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294022:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.294031:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.294033:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.294037:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.294040:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24376, 24376] 00000080:00200000:0.0:1775764896.294051:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.294054:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.294081:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99848192 i_blocks 90112 00000080:00200000:0.0:1775764896.294086:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.294101:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99848192, count: 4096 00000080:00200000:0.0:1775764896.294125:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99848192, bytes: 4096 00000080:00200000:0.0:1775764896.294128:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.294132:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24378, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.294134:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99848192, 99852288) 0 0 00000080:00200000:0.0:1775764896.294136:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.294141:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24377, 24377] 00020000:00200000:0.0:1775764896.294144:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99848192 chunk: [99848192, 99852288) 99852288, 4096 00020000:00200000:0.0:1775764896.294146:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.294159:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24377, 24377] 00000008:00010000:0.0:1775764896.294164:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.294167:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.294174:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294182:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99848192 99852287) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294190:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294203:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294210:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.294218:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99848192, 99852288) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.294223:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24377 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.294239:0:18302:0:(rw26.c:858:ll_write_end()) pos 99848192, len 4096, copied 4096 00000080:00200000:0.0:1775764896.294243:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a875c90e[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.294245:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a08242e9 17ffffc0001001 3:0 ffff90c212831620 24377 lru 00000080:00200000:0.0:1775764896.294251:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007d33c327 24377: 1< 0 - - > 2< 99848192 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 765952 - | + - + - > 5< + - + - | 0 - | 3897 - -> 00000080:00200000:0.0:1775764896.294252:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a875c90e 00000080:00200000:0.0:1775764896.294253:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.294255:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.294262:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.294270:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.294276:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.294283:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.294300:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.294303:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.294305:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.294314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.294317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.294319:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.294323:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294330:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.294338:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.294340:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.294344:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.294347:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24377, 24377] 00000080:00200000:0.0:1775764896.294359:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.294361:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.294391:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99852288 i_blocks 90112 00000080:00200000:0.0:1775764896.294395:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.294410:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99852288, count: 4096 00000080:00200000:0.0:1775764896.294420:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99852288, bytes: 4096 00000080:00200000:0.0:1775764896.294422:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.294425:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24379, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.294427:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99852288, 99856384) 0 0 00000080:00200000:0.0:1775764896.294429:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.294434:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24378, 24378] 00020000:00200000:0.0:1775764896.294437:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99852288 chunk: [99852288, 99856384) 99856384, 4096 00020000:00200000:0.0:1775764896.294440:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.294451:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24378, 24378] 00000008:00010000:0.0:1775764896.294457:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.294459:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.294466:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294474:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99852288 99856383) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294481:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294487:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294493:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.294500:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99852288, 99856384) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.294505:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24378 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.294520:0:18302:0:(rw26.c:858:ll_write_end()) pos 99852288, len 4096, copied 4096 00000080:00200000:0.0:1775764896.294524:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006329cd1f[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.294526:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009ef9cc0e 17ffffc0001001 3:0 ffff90c212831e00 24378 lru 00000080:00200000:0.0:1775764896.294532:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c5384fb7 24378: 1< 0 - - > 2< 99852288 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 761856 - | + - + - > 5< + - + - | 0 - | 3898 - -> 00000080:00200000:0.0:1775764896.294534:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006329cd1f 00000080:00200000:0.0:1775764896.294535:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.294537:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.294545:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.294552:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.294559:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.294565:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.294575:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.294578:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.294580:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.294590:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.294592:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.294594:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.294598:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294605:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.294613:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.294616:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.294620:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.294623:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24378, 24378] 00000080:00200000:0.0:1775764896.294634:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.294637:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.294666:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99856384 i_blocks 90112 00000080:00200000:0.0:1775764896.294670:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.294686:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99856384, count: 4096 00000080:00200000:0.0:1775764896.294696:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99856384, bytes: 4096 00000080:00200000:0.0:1775764896.294699:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.294703:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24380, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.294704:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99856384, 99860480) 0 0 00000080:00200000:0.0:1775764896.294707:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.294712:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24379, 24379] 00020000:00200000:0.0:1775764896.294715:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99856384 chunk: [99856384, 99860480) 99860480, 4096 00020000:00200000:0.0:1775764896.294718:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.294730:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24379, 24379] 00000008:00010000:0.0:1775764896.294735:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.294737:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.294744:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294753:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99856384 99860479) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294762:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294770:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.294959:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.294969:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99856384, 99860480) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.294976:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24379 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.294995:0:18302:0:(rw26.c:858:ll_write_end()) pos 99856384, len 4096, copied 4096 00000080:00200000:0.0:1775764896.294999:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b7250870[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.295002:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000dfdbfaee 17ffffc0001001 3:0 ffff90c212831540 24379 lru 00000080:00200000:0.0:1775764896.295007:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008290f909 24379: 1< 0 - - > 2< 99856384 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 757760 - | + - + - > 5< + - + - | 0 - | 3899 - -> 00000080:00200000:0.0:1775764896.295009:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b7250870 00000080:00200000:0.0:1775764896.295011:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.295013:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.295021:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.295028:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.295034:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.295041:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.295051:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.295054:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.295056:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.295067:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.295069:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.295071:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.295075:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295082:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.295089:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.295091:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.295095:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.295098:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24379, 24379] 00000080:00200000:0.0:1775764896.295108:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.295123:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.295156:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99860480 i_blocks 90112 00000080:00200000:0.0:1775764896.295159:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.295176:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99860480, count: 4096 00000080:00200000:0.0:1775764896.295186:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99860480, bytes: 4096 00000080:00200000:0.0:1775764896.295188:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.295190:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24381, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.295191:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99860480, 99864576) 0 0 00000080:00200000:0.0:1775764896.295193:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.295197:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24380, 24380] 00020000:00200000:0.0:1775764896.295199:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99860480 chunk: [99860480, 99864576) 99864576, 4096 00020000:00200000:0.0:1775764896.295201:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.295208:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24380, 24380] 00000008:00010000:0.0:1775764896.295212:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.295213:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.295219:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295226:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99860480 99864575) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295232:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295237:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295240:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.295245:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99860480, 99864576) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.295248:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24380 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.295260:0:18302:0:(rw26.c:858:ll_write_end()) pos 99860480, len 4096, copied 4096 00000080:00200000:0.0:1775764896.295263:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000023f3b213[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.295265:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000f42d650 17ffffc0001001 3:0 ffff90c212831460 24380 lru 00000080:00200000:0.0:1775764896.295268:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008aa5d173 24380: 1< 0 - - > 2< 99860480 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 753664 - | + - + - > 5< + - + - | 0 - | 3900 - -> 00000080:00200000:0.0:1775764896.295270:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000023f3b213 00000080:00200000:0.0:1775764896.295271:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.295272:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.295278:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.295283:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.295287:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.295295:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.295302:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.295304:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.295305:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.295313:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.295315:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.295317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.295321:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295328:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.295337:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.295339:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.295343:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.295346:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24380, 24380] 00000080:00200000:0.0:1775764896.295357:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.295360:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.295384:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99864576 i_blocks 90112 00000080:00200000:0.0:1775764896.295386:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.295397:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99864576, count: 4096 00000080:00200000:0.0:1775764896.295403:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99864576, bytes: 4096 00000080:00200000:0.0:1775764896.295405:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.295408:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24382, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.295409:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99864576, 99868672) 0 0 00000080:00200000:0.0:1775764896.295411:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.295415:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24381, 24381] 00020000:00200000:0.0:1775764896.295417:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99864576 chunk: [99864576, 99868672) 99868672, 4096 00020000:00200000:0.0:1775764896.295418:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.295425:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24381, 24381] 00000008:00010000:0.0:1775764896.295428:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.295430:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.295435:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295441:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99864576 99868671) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295446:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295451:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295454:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.295459:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99864576, 99868672) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.295462:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24381 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.295474:0:18302:0:(rw26.c:858:ll_write_end()) pos 99864576, len 4096, copied 4096 00000080:00200000:0.0:1775764896.295477:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e6d283ff[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.295480:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e376e4fa 17ffffc0001001 3:0 ffff90c212831a80 24381 lru 00000080:00200000:0.0:1775764896.295485:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001c157e15 24381: 1< 0 - - > 2< 99864576 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 749568 - | + - + - > 5< + - + - | 0 - | 3901 - -> 00000080:00200000:0.0:1775764896.295487:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e6d283ff 00000080:00200000:0.0:1775764896.295489:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.295491:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.295499:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.295506:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.295525:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.295534:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.295545:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.295549:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.295551:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.295560:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.295563:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.295565:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.295568:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295576:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.295584:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.295587:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.295591:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.295595:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24381, 24381] 00000080:00200000:0.0:1775764896.295605:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.295608:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.295629:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99868672 i_blocks 90112 00000080:00200000:0.0:1775764896.295631:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.295643:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99868672, count: 4096 00000080:00200000:0.0:1775764896.295649:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99868672, bytes: 4096 00000080:00200000:0.0:1775764896.295650:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.295653:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24383, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.295654:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99868672, 99872768) 0 0 00000080:00200000:0.0:1775764896.295656:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.295660:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24382, 24382] 00020000:00200000:0.0:1775764896.295662:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99868672 chunk: [99868672, 99872768) 99872768, 4096 00020000:00200000:0.0:1775764896.295663:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.295670:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24382, 24382] 00000008:00010000:0.0:1775764896.295675:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.295678:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.295685:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295693:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99868672 99872767) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295701:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295708:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.295721:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.295730:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99868672, 99872768) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.295735:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24382 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.295750:0:18302:0:(rw26.c:858:ll_write_end()) pos 99868672, len 4096, copied 4096 00000080:00200000:0.0:1775764896.295753:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000090207a94[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.295755:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c3169d03 17ffffc0001001 3:0 ffff90c212831700 24382 lru 00000080:00200000:0.0:1775764896.295759:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c4b06fca 24382: 1< 0 - - > 2< 99868672 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 745472 - | + - + - > 5< + - + - | 0 - | 3902 - -> 00000080:00200000:0.0:1775764896.295760:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000090207a94 00000080:00200000:0.0:1775764896.295761:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.295763:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.295768:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296075:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296080:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296085:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.296094:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.296097:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.296099:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.296106:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.296108:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.296110:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.296123:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296131:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.296137:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.296138:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.296142:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.296145:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24382, 24382] 00000080:00200000:0.0:1775764896.296152:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.296154:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.296176:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99872768 i_blocks 90112 00000080:00200000:0.0:1775764896.296179:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.296190:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99872768, count: 4096 00000080:00200000:0.0:1775764896.296197:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99872768, bytes: 4096 00000080:00200000:0.0:1775764896.296199:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.296201:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24384, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.296202:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99872768, 99876864) 0 0 00000080:00200000:0.0:1775764896.296204:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.296207:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24383, 24383] 00020000:00200000:0.0:1775764896.296209:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99872768 chunk: [99872768, 99876864) 99876864, 4096 00020000:00200000:0.0:1775764896.296211:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.296218:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24383, 24383] 00000008:00010000:0.0:1775764896.296221:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.296224:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.296229:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296235:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99872768 99876863) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296241:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296245:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296249:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.296253:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99872768, 99876864) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.296256:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24383 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.296269:0:18302:0:(rw26.c:858:ll_write_end()) pos 99872768, len 4096, copied 4096 00000080:00200000:0.0:1775764896.296272:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a99e3026[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.296274:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ab9810a3 17ffffc0001001 3:0 ffff90c212831c40 24383 lru 00000080:00200000:0.0:1775764896.296278:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000937bb331 24383: 1< 0 - - > 2< 99872768 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 741376 - | + - + - > 5< + - + - | 0 - | 3903 - -> 00000080:00200000:0.0:1775764896.296279:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a99e3026 00000080:00200000:0.0:1775764896.296280:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.296281:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.296286:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296291:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296296:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296300:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.296307:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.296308:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.296310:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.296316:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.296317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.296319:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.296321:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296326:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.296331:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.296332:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.296334:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.296336:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24383, 24383] 00000080:00200000:0.0:1775764896.296343:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.296345:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.296364:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99876864 i_blocks 90112 00000080:00200000:0.0:1775764896.296366:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.296377:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99876864, count: 4096 00000080:00200000:0.0:1775764896.296383:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99876864, bytes: 4096 00000080:00200000:0.0:1775764896.296384:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.296387:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24385, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.296388:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99876864, 99880960) 0 0 00000080:00200000:0.0:1775764896.296389:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.296393:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24384, 24384] 00020000:00200000:0.0:1775764896.296395:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99876864 chunk: [99876864, 99880960) 99880960, 4096 00020000:00200000:0.0:1775764896.296396:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.296402:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24384, 24384] 00000008:00010000:0.0:1775764896.296405:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.296407:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.296412:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296418:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99876864 99880959) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296423:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296428:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296431:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.296435:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99876864, 99880960) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.296438:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24384 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.296471:0:18302:0:(rw26.c:858:ll_write_end()) pos 99876864, len 4096, copied 4096 00000080:00200000:0.0:1775764896.296474:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b1fa154c[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.296476:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d6986e89 17ffffc0001001 3:0 ffff90c2128318c0 24384 lru 00000080:00200000:0.0:1775764896.296481:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000747bf55f 24384: 1< 0 - - > 2< 99876864 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 737280 - | + - + - > 5< + - + - | 0 - | 3904 - -> 00000080:00200000:0.0:1775764896.296484:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b1fa154c 00000080:00200000:0.0:1775764896.296485:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.296487:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.296494:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296502:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296509:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296517:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.296528:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.296531:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.296532:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.296540:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.296543:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.296546:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.296549:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296557:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.296565:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.296568:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.296572:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.296574:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24384, 24384] 00000080:00200000:0.0:1775764896.296585:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.296589:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.296616:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99880960 i_blocks 90112 00000080:00200000:0.0:1775764896.296619:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.296630:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99880960, count: 4096 00000080:00200000:0.0:1775764896.296637:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99880960, bytes: 4096 00000080:00200000:0.0:1775764896.296639:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.296641:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24386, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.296643:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99880960, 99885056) 0 0 00000080:00200000:0.0:1775764896.296644:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.296648:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24385, 24385] 00020000:00200000:0.0:1775764896.296650:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99880960 chunk: [99880960, 99885056) 99885056, 4096 00020000:00200000:0.0:1775764896.296651:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.296658:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24385, 24385] 00000008:00010000:0.0:1775764896.296661:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.296663:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.296668:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296676:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99880960 99885055) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296684:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296695:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296702:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.296710:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99880960, 99885056) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.296714:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24385 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.296731:0:18302:0:(rw26.c:858:ll_write_end()) pos 99880960, len 4096, copied 4096 00000080:00200000:0.0:1775764896.296735:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000363774c4[3 00000000b99c1f5d 2 1 00000000c7bb9423] 00000080:00200000:0.0:1775764896.296738:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000021d72930 17ffffc0001001 3:0 ffff90c212831b60 24385 lru 00000080:00200000:0.0:1775764896.296742:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c5337161 24385: 1< 0 - - > 2< 99880960 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 733184 - | + - + - > 5< + - + - | 0 - | 3905 - -> 00000080:00200000:0.0:1775764896.296744:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000363774c4 00000080:00200000:0.0:1775764896.296745:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.296747:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.296752:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296758:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000009209349b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296761:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.296765:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.296848:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.296851:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.296852:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.296861:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.296863:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.296866:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.296928:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.296935:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.296941:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a17b8 (ffff90c210c815a8/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.296942:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c815a8/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.296945:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.296947:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24385, 24385] 00000080:00200000:0.0:1775764896.296955:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.296957:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.297052:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99885056 i_blocks 90112 00000080:00200000:1.0:1775764896.297055:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.297071:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99885056, count: 4096 00000080:00200000:1.0:1775764896.297084:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99885056, bytes: 4096 00000080:00200000:1.0:1775764896.297086:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.297088:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24387, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.297089:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99885056, 99889152) 0 0 00000080:00200000:1.0:1775764896.297091:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.297094:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24386, 24386] 00020000:00200000:1.0:1775764896.297097:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99885056 chunk: [99885056, 99889152) 99889152, 4096 00020000:00200000:1.0:1775764896.297098:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.297109:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24386, 24386] 00000008:00010000:1.0:1775764896.297123:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.297126:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.297132:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297138:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99885056 99889151) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297144:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297149:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297152:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.297158:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99885056, 99889152) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.297161:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24386 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.297177:0:18302:0:(rw26.c:858:ll_write_end()) pos 99885056, len 4096, copied 4096 00000080:00200000:1.0:1775764896.297181:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e44c1859[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.297184:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000873f8c14 17ffffc0001001 3:0 ffff90c21282ab60 24386 lru 00000080:00200000:1.0:1775764896.297189:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000021b47ea5 24386: 1< 0 - - > 2< 99885056 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 729088 - | + - + - > 5< + - + - | 0 - | 3906 - -> 00000080:00200000:1.0:1775764896.297191:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e44c1859 00000080:00200000:1.0:1775764896.297192:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.297194:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.297202:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.297210:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.297217:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.297224:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.297233:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.297235:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.297237:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.297245:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.297247:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.297249:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.297251:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297256:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.297262:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.297264:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.297268:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.297270:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24386, 24386] 00000080:00200000:1.0:1775764896.297286:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.297290:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.297319:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99889152 i_blocks 90112 00000080:00200000:1.0:1775764896.297322:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.297339:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99889152, count: 4096 00000080:00200000:1.0:1775764896.297349:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99889152, bytes: 4096 00000080:00200000:1.0:1775764896.297352:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.297356:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24388, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.297358:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99889152, 99893248) 0 0 00000080:00200000:1.0:1775764896.297360:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.297365:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24387, 24387] 00020000:00200000:1.0:1775764896.297368:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99889152 chunk: [99889152, 99893248) 99893248, 4096 00020000:00200000:1.0:1775764896.297370:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.297380:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24387, 24387] 00000008:00010000:1.0:1775764896.297385:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.297387:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.297395:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297404:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99889152 99893247) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297411:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297418:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297424:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.297436:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99889152, 99893248) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.297441:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24387 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.297465:0:18302:0:(rw26.c:858:ll_write_end()) pos 99889152, len 4096, copied 4096 00000080:00200000:1.0:1775764896.297469:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000634e4d7c[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.297472:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002c375ad7 17ffffc0001001 3:0 ffff90c212834e00 24387 lru 00000080:00200000:1.0:1775764896.297476:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000e86f0e2 24387: 1< 0 - - > 2< 99889152 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 724992 - | + - + - > 5< + - + - | 0 - | 3907 - -> 00000080:00200000:1.0:1775764896.297479:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000634e4d7c 00000080:00200000:1.0:1775764896.297480:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.297482:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.297490:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.297498:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.297505:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.297512:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.297523:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.297526:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.297528:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.297538:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.297541:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.297544:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.297548:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297555:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.297563:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.297566:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.297570:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.297573:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24387, 24387] 00000080:00200000:1.0:1775764896.297584:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.297587:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.297617:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99893248 i_blocks 90112 00000080:00200000:1.0:1775764896.297621:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.297638:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99893248, count: 4096 00000080:00200000:1.0:1775764896.297648:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99893248, bytes: 4096 00000080:00200000:1.0:1775764896.297651:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.297655:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24389, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.297656:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99893248, 99897344) 0 0 00000080:00200000:1.0:1775764896.297659:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.297664:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24388, 24388] 00020000:00200000:1.0:1775764896.297667:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99893248 chunk: [99893248, 99897344) 99897344, 4096 00020000:00200000:1.0:1775764896.297669:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.297680:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24388, 24388] 00000008:00010000:1.0:1775764896.297686:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.297688:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.297696:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297705:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99893248 99897343) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297714:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297720:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297727:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.297736:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99893248, 99897344) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.297741:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24388 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.297758:0:18302:0:(rw26.c:858:ll_write_end()) pos 99893248, len 4096, copied 4096 00000080:00200000:1.0:1775764896.297763:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000089d06063[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.297766:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000136cef84 17ffffc0001001 3:0 ffff90c212834540 24388 lru 00000080:00200000:1.0:1775764896.297891:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cc3efe84 24388: 1< 0 - - > 2< 99893248 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 720896 - | + - + - > 5< + - + - | 0 - | 3908 - -> 00000080:00200000:1.0:1775764896.297895:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000089d06063 00000080:00200000:1.0:1775764896.297896:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.297899:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.297908:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.297915:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.297921:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.297934:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.297946:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.297949:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.297951:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.297960:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.297962:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.297965:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.297969:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.297977:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.297986:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.297989:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.297993:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.297995:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24388, 24388] 00000080:00200000:1.0:1775764896.298006:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.298009:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.298038:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99897344 i_blocks 90112 00000080:00200000:1.0:1775764896.298042:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.298058:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99897344, count: 4096 00000080:00200000:1.0:1775764896.298068:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99897344, bytes: 4096 00000080:00200000:1.0:1775764896.298070:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.298074:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24390, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.298076:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99897344, 99901440) 0 0 00000080:00200000:1.0:1775764896.298079:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.298084:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24389, 24389] 00020000:00200000:1.0:1775764896.298087:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99897344 chunk: [99897344, 99901440) 99901440, 4096 00020000:00200000:1.0:1775764896.298089:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.298100:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24389, 24389] 00000008:00010000:1.0:1775764896.298106:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.298109:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.298128:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298138:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99897344 99901439) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298146:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298154:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298161:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.298169:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99897344, 99901440) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.298175:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24389 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.298190:0:18302:0:(rw26.c:858:ll_write_end()) pos 99897344, len 4096, copied 4096 00000080:00200000:1.0:1775764896.298194:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003f3c0797[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.298197:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009bd397fe 17ffffc0001001 3:0 ffff90c212834460 24389 lru 00000080:00200000:1.0:1775764896.298203:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000357f083b 24389: 1< 0 - - > 2< 99897344 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 716800 - | + - + - > 5< + - + - | 0 - | 3909 - -> 00000080:00200000:1.0:1775764896.298206:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003f3c0797 00000080:00200000:1.0:1775764896.298207:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.298209:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.298217:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.298225:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.298233:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.298241:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.298253:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.298256:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.298258:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.298267:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.298270:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.298272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.298276:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298285:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.298293:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.298297:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.298301:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.298304:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24389, 24389] 00000080:00200000:1.0:1775764896.298314:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.298317:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.298345:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99901440 i_blocks 90112 00000080:00200000:1.0:1775764896.298349:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.298365:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99901440, count: 4096 00000080:00200000:1.0:1775764896.298375:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99901440, bytes: 4096 00000080:00200000:1.0:1775764896.298378:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.298382:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24391, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.298383:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99901440, 99905536) 0 0 00000080:00200000:1.0:1775764896.298386:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.298391:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24390, 24390] 00020000:00200000:1.0:1775764896.298394:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99901440 chunk: [99901440, 99905536) 99905536, 4096 00020000:00200000:1.0:1775764896.298397:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.298409:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24390, 24390] 00000008:00010000:1.0:1775764896.298415:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.298417:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.298425:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298434:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99901440 99905535) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298442:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298449:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298462:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.298470:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99901440, 99905536) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.298475:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24390 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.298492:0:18302:0:(rw26.c:858:ll_write_end()) pos 99901440, len 4096, copied 4096 00000080:00200000:1.0:1775764896.298496:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b8a57f6f[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.298499:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000246265ba 17ffffc0001001 3:0 ffff90c212834a80 24390 lru 00000080:00200000:1.0:1775764896.298504:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000893a0f1d 24390: 1< 0 - - > 2< 99901440 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 712704 - | + - + - > 5< + - + - | 0 - | 3910 - -> 00000080:00200000:1.0:1775764896.298506:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b8a57f6f 00000080:00200000:1.0:1775764896.298507:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.298509:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.298517:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.298524:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.298531:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.298539:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.298550:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.298553:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.298556:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.298565:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.298567:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.298570:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.298574:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298582:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.298590:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.298593:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.298597:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.298600:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24390, 24390] 00000080:00200000:1.0:1775764896.298610:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.298613:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.298641:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99905536 i_blocks 90112 00000080:00200000:1.0:1775764896.298645:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.298660:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99905536, count: 4096 00000080:00200000:1.0:1775764896.298669:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99905536, bytes: 4096 00000080:00200000:1.0:1775764896.298672:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.298676:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24392, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.298677:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99905536, 99909632) 0 0 00000080:00200000:1.0:1775764896.298679:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.298684:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24391, 24391] 00020000:00200000:1.0:1775764896.298687:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99905536 chunk: [99905536, 99909632) 99909632, 4096 00020000:00200000:1.0:1775764896.298690:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.298701:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24391, 24391] 00000008:00010000:1.0:1775764896.298706:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.298709:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.298717:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298726:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99905536 99909631) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298735:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298743:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.298749:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.298757:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99905536, 99909632) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.298762:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24391 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.298969:0:18302:0:(rw26.c:858:ll_write_end()) pos 99905536, len 4096, copied 4096 00000080:00200000:1.0:1775764896.298974:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005125d3cd[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.298976:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ddf80399 17ffffc0001001 3:0 ffff90c212834700 24391 lru 00000080:00200000:1.0:1775764896.298981:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c089bef4 24391: 1< 0 - - > 2< 99905536 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 708608 - | + - + - > 5< + - + - | 0 - | 3911 - -> 00000080:00200000:1.0:1775764896.298983:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005125d3cd 00000080:00200000:1.0:1775764896.298984:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.298986:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.298994:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299001:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299008:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299014:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.299025:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.299028:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.299030:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.299041:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.299043:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.299045:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.299049:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299055:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.299063:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.299065:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.299069:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.299072:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24391, 24391] 00000080:00200000:1.0:1775764896.299082:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.299085:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.299126:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99909632 i_blocks 90112 00000080:00200000:1.0:1775764896.299130:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.299148:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99909632, count: 4096 00000080:00200000:1.0:1775764896.299160:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99909632, bytes: 4096 00000080:00200000:1.0:1775764896.299162:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.299166:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24393, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.299168:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99909632, 99913728) 0 0 00000080:00200000:1.0:1775764896.299170:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.299176:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24392, 24392] 00020000:00200000:1.0:1775764896.299179:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99909632 chunk: [99909632, 99913728) 99913728, 4096 00020000:00200000:1.0:1775764896.299181:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.299193:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24392, 24392] 00000008:00010000:1.0:1775764896.299197:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.299200:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.299207:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299216:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99909632 99913727) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299225:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299233:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299240:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.299248:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99909632, 99913728) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.299253:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24392 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.299270:0:18302:0:(rw26.c:858:ll_write_end()) pos 99909632, len 4096, copied 4096 00000080:00200000:1.0:1775764896.299275:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c19e02db[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.299277:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002e1df8ca 17ffffc0001001 3:0 ffff90c212834c40 24392 lru 00000080:00200000:1.0:1775764896.299283:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005c79fa54 24392: 1< 0 - - > 2< 99909632 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 704512 - | + - + - > 5< + - + - | 0 - | 3912 - -> 00000080:00200000:1.0:1775764896.299285:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c19e02db 00000080:00200000:1.0:1775764896.299286:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.299288:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.299296:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299305:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299312:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299320:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.299332:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.299335:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.299337:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.299348:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.299350:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.299353:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.299357:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299364:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.299373:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.299375:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.299380:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.299383:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24392, 24392] 00000080:00200000:1.0:1775764896.299393:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.299396:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.299425:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99913728 i_blocks 90112 00000080:00200000:1.0:1775764896.299428:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.299444:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99913728, count: 4096 00000080:00200000:1.0:1775764896.299455:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99913728, bytes: 4096 00000080:00200000:1.0:1775764896.299458:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.299461:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24394, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.299463:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99913728, 99917824) 0 0 00000080:00200000:1.0:1775764896.299465:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.299470:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24393, 24393] 00020000:00200000:1.0:1775764896.299473:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99913728 chunk: [99913728, 99917824) 99917824, 4096 00020000:00200000:1.0:1775764896.299476:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.299487:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24393, 24393] 00000008:00010000:1.0:1775764896.299493:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.299495:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.299503:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299511:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99913728 99917823) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299519:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299533:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299540:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.299548:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99913728, 99917824) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.299553:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24393 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.299571:0:18302:0:(rw26.c:858:ll_write_end()) pos 99913728, len 4096, copied 4096 00000080:00200000:1.0:1775764896.299575:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000073f5ed79[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.299578:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000005262be4 17ffffc0001001 3:0 ffff90c2128348c0 24393 lru 00000080:00200000:1.0:1775764896.299584:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ca5eb450 24393: 1< 0 - - > 2< 99913728 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 700416 - | + - + - > 5< + - + - | 0 - | 3913 - -> 00000080:00200000:1.0:1775764896.299586:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000073f5ed79 00000080:00200000:1.0:1775764896.299587:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.299589:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.299597:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299604:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299611:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299618:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.299636:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.299639:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.299641:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.299652:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.299655:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.299657:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.299660:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299667:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.299676:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.299678:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.299683:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.299686:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24393, 24393] 00000080:00200000:1.0:1775764896.299697:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.299700:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.299729:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99917824 i_blocks 90112 00000080:00200000:1.0:1775764896.299733:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.299749:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99917824, count: 4096 00000080:00200000:1.0:1775764896.299760:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99917824, bytes: 4096 00000080:00200000:1.0:1775764896.299763:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.299767:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24395, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.299768:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99917824, 99921920) 0 0 00000080:00200000:1.0:1775764896.299771:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.299853:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24394, 24394] 00020000:00200000:1.0:1775764896.299856:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99917824 chunk: [99917824, 99921920) 99921920, 4096 00020000:00200000:1.0:1775764896.299857:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.299867:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24394, 24394] 00000008:00010000:1.0:1775764896.299888:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.299890:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.299896:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299902:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99917824 99921919) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299908:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299912:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299916:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.299921:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99917824, 99921920) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.299925:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24394 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.299939:0:18302:0:(rw26.c:858:ll_write_end()) pos 99917824, len 4096, copied 4096 00000080:00200000:1.0:1775764896.299942:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e75779a2[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.299944:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000038e6bc0 17ffffc0001001 3:0 ffff90c212834b60 24394 lru 00000080:00200000:1.0:1775764896.299948:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000019c1456b 24394: 1< 0 - - > 2< 99917824 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 696320 - | + - + - > 5< + - + - | 0 - | 3914 - -> 00000080:00200000:1.0:1775764896.299949:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e75779a2 00000080:00200000:1.0:1775764896.299950:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.299951:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.299957:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299962:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299967:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.299971:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.299978:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.299979:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.299981:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.299987:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.299989:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.299991:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.299993:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.299998:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.300003:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.300004:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.300007:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.300009:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24394, 24394] 00000080:00200000:1.0:1775764896.300016:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.300018:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.300036:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99921920 i_blocks 90112 00000080:00200000:1.0:1775764896.300039:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.300050:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99921920, count: 4096 00000080:00200000:1.0:1775764896.300056:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99921920, bytes: 4096 00000080:00200000:1.0:1775764896.300058:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.300060:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24396, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.300061:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99921920, 99926016) 0 0 00000080:00200000:1.0:1775764896.300063:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.300066:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24395, 24395] 00020000:00200000:1.0:1775764896.300068:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99921920 chunk: [99921920, 99926016) 99926016, 4096 00020000:00200000:1.0:1775764896.300069:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.300076:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24395, 24395] 00000008:00010000:1.0:1775764896.300079:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.300080:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.300085:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300091:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99921920 99926015) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300096:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300100:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300104:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.300109:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99921920, 99926016) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.300124:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24395 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.300138:0:18302:0:(rw26.c:858:ll_write_end()) pos 99921920, len 4096, copied 4096 00000080:00200000:1.0:1775764896.300141:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000bad6ad38[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.300143:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e5875207 17ffffc0001001 3:0 ffff90c212834000 24395 lru 00000080:00200000:1.0:1775764896.300147:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009a79e972 24395: 1< 0 - - > 2< 99921920 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 692224 - | + - + - > 5< + - + - | 0 - | 3915 - -> 00000080:00200000:1.0:1775764896.300148:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000bad6ad38 00000080:00200000:1.0:1775764896.300149:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.300151:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.300156:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.300161:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.300165:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.300169:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.300175:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.300177:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.300178:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.300185:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.300186:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.300188:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.300190:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300195:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.300201:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.300202:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.300205:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.300207:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24395, 24395] 00000080:00200000:1.0:1775764896.300214:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.300216:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.300234:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99926016 i_blocks 90112 00000080:00200000:1.0:1775764896.300236:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.300249:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99926016, count: 4096 00000080:00200000:1.0:1775764896.300259:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99926016, bytes: 4096 00000080:00200000:1.0:1775764896.300262:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.300265:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24397, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.300267:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99926016, 99930112) 0 0 00000080:00200000:1.0:1775764896.300269:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.300274:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24396, 24396] 00020000:00200000:1.0:1775764896.300277:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99926016 chunk: [99926016, 99930112) 99930112, 4096 00020000:00200000:1.0:1775764896.300279:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.300291:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24396, 24396] 00000008:00010000:1.0:1775764896.300296:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.300299:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.300306:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300315:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99926016 99930111) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300324:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300332:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300338:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.300344:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99926016, 99930112) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.300348:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24396 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.300361:0:18302:0:(rw26.c:858:ll_write_end()) pos 99926016, len 4096, copied 4096 00000080:00200000:1.0:1775764896.300363:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000fa051ed[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.300365:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b0f60cc8 17ffffc0001001 3:0 ffff90c212834ee0 24396 lru 00000080:00200000:1.0:1775764896.300369:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000dd81dabc 24396: 1< 0 - - > 2< 99926016 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 688128 - | + - + - > 5< + - + - | 0 - | 3916 - -> 00000080:00200000:1.0:1775764896.300370:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000fa051ed 00000080:00200000:1.0:1775764896.300371:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.300372:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.300377:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.300383:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.300390:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.300402:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.300413:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.300416:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.300418:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.300428:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.300431:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.300434:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.300437:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300445:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.300453:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.300456:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.300459:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.300462:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24396, 24396] 00000080:00200000:1.0:1775764896.300472:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.300474:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.300503:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99930112 i_blocks 90112 00000080:00200000:1.0:1775764896.300507:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.300523:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99930112, count: 4096 00000080:00200000:1.0:1775764896.300533:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99930112, bytes: 4096 00000080:00200000:1.0:1775764896.300536:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.300540:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24398, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.300542:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99930112, 99934208) 0 0 00000080:00200000:1.0:1775764896.300544:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.300550:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24397, 24397] 00020000:00200000:1.0:1775764896.300553:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99930112 chunk: [99930112, 99934208) 99934208, 4096 00020000:00200000:1.0:1775764896.300555:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.300566:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24397, 24397] 00000008:00010000:1.0:1775764896.300571:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.300573:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.300581:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300590:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99930112 99934207) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300599:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300606:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300613:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.300621:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99930112, 99934208) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.300627:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24397 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.300643:0:18302:0:(rw26.c:858:ll_write_end()) pos 99930112, len 4096, copied 4096 00000080:00200000:1.0:1775764896.300647:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a4691f99[3 00000000b99c1f5d 2 1 000000009b51b744] 00000080:00200000:1.0:1775764896.300650:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000079a4d9f9 17ffffc0001001 3:0 ffff90c2128340e0 24397 lru 00000080:00200000:1.0:1775764896.300656:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ce7d2a34 24397: 1< 0 - - > 2< 99930112 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 684032 - | + - + - > 5< + - + - | 0 - | 3917 - -> 00000080:00200000:1.0:1775764896.300658:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a4691f99 00000080:00200000:1.0:1775764896.300659:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.300661:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.300668:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.300676:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000067ed64c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.300682:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.300689:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.300700:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.300703:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.300704:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.300714:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.300716:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.300719:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.300723:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.300731:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.300740:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea308 (ffff90c2195ee5a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.300742:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee5a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.300746:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.300749:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24397, 24397] 00000080:00200000:1.0:1775764896.300760:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.300763:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.300957:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99934208 i_blocks 90112 00000080:00200000:0.0:1775764896.300962:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.300983:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99934208, count: 4096 00000080:00200000:0.0:1775764896.301000:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99934208, bytes: 4096 00000080:00200000:0.0:1775764896.301003:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.301008:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24399, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.301009:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99934208, 99938304) 0 0 00000080:00200000:0.0:1775764896.301012:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.301017:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24398, 24398] 00020000:00200000:0.0:1775764896.301020:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99934208 chunk: [99934208, 99938304) 99938304, 4096 00020000:00200000:0.0:1775764896.301023:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.301037:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24398, 24398] 00000008:00010000:0.0:1775764896.301044:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.301047:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.301055:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301064:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99934208 99938303) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301073:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301082:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301089:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.301096:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99934208, 99938304) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.301101:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24398 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.301136:0:18302:0:(rw26.c:858:ll_write_end()) pos 99934208, len 4096, copied 4096 00000080:00200000:0.0:1775764896.301141:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b311305d[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.301143:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c9a04612 17ffffc0001021 2:0 ffff90c212831000 24398 lru 00000080:00200000:0.0:1775764896.301149:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b7fb7951 24398: 1< 0 - - > 2< 99934208 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 679936 - | + - + - > 5< + - + - | 0 - | 3918 - -> 00000080:00200000:0.0:1775764896.301150:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b311305d 00000080:00200000:0.0:1775764896.301151:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.301154:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.301162:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301171:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301179:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301187:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.301198:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.301201:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.301203:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.301210:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.301213:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.301215:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.301219:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301227:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.301236:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.301239:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.301244:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.301247:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24398, 24398] 00000080:00200000:0.0:1775764896.301260:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.301264:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.301268:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99938304 i_blocks 90112 00000080:00200000:0.0:1775764896.301271:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.301290:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99938304, count: 4096 00000080:00200000:0.0:1775764896.301301:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99938304, bytes: 4096 00000080:00200000:0.0:1775764896.301304:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.301308:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24400, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.301310:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99938304, 99942400) 0 0 00000080:00200000:0.0:1775764896.301313:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.301318:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24399, 24399] 00020000:00200000:0.0:1775764896.301321:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99938304 chunk: [99938304, 99942400) 99942400, 4096 00020000:00200000:0.0:1775764896.301323:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.301335:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24399, 24399] 00000008:00010000:0.0:1775764896.301339:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.301342:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.301350:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301359:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99938304 99942399) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301368:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301374:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301380:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.301388:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99938304, 99942400) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.301392:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24399 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.301408:0:18302:0:(rw26.c:858:ll_write_end()) pos 99938304, len 4096, copied 4096 00000080:00200000:0.0:1775764896.301412:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000e1ff2b0[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.301415:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a1da2745 17ffffc0001001 3:0 ffff90c212831ee0 24399 lru 00000080:00200000:0.0:1775764896.301420:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005ae40da9 24399: 1< 0 - - > 2< 99938304 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 675840 - | + - + - > 5< + - + - | 0 - | 3919 - -> 00000080:00200000:0.0:1775764896.301423:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000e1ff2b0 00000080:00200000:0.0:1775764896.301424:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.301426:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.301433:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301440:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301447:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301455:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.301465:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.301468:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.301469:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.301475:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.301477:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.301480:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.301484:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301492:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.301500:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.301502:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.301506:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.301509:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24399, 24399] 00000080:00200000:0.0:1775764896.301518:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.301521:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.301526:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99942400 i_blocks 90112 00000080:00200000:0.0:1775764896.301528:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.301543:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99942400, count: 4096 00000080:00200000:0.0:1775764896.301554:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99942400, bytes: 4096 00000080:00200000:0.0:1775764896.301557:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.301561:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24401, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.301562:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99942400, 99946496) 0 0 00000080:00200000:0.0:1775764896.301565:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.301569:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24400, 24400] 00020000:00200000:0.0:1775764896.301572:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99942400 chunk: [99942400, 99946496) 99946496, 4096 00020000:00200000:0.0:1775764896.301575:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.301586:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24400, 24400] 00000008:00010000:0.0:1775764896.301591:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.301594:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.301601:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301610:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99942400 99946495) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301619:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301628:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301635:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.301644:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99942400, 99946496) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.301648:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24400 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.301664:0:18302:0:(rw26.c:858:ll_write_end()) pos 99942400, len 4096, copied 4096 00000080:00200000:0.0:1775764896.301669:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e701dcfc[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.301671:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f0fff6ec 17ffffc0001001 3:0 ffff90c2128310e0 24400 lru 00000080:00200000:0.0:1775764896.301677:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f886d6f2 24400: 1< 0 - - > 2< 99942400 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 671744 - | + - + - > 5< + - + - | 0 - | 3920 - -> 00000080:00200000:0.0:1775764896.301679:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e701dcfc 00000080:00200000:0.0:1775764896.301680:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.301682:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.301690:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301699:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301706:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301720:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.301730:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.301733:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.301735:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.301740:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.301742:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.301745:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.301748:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301756:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.301765:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.301767:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.301784:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.301788:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24400, 24400] 00000080:00200000:0.0:1775764896.301798:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.301802:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.301806:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99946496 i_blocks 90112 00000080:00200000:0.0:1775764896.301809:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.301824:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99946496, count: 4096 00000080:00200000:0.0:1775764896.301835:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99946496, bytes: 4096 00000080:00200000:0.0:1775764896.301838:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.301843:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24402, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.301844:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99946496, 99950592) 0 0 00000080:00200000:0.0:1775764896.301847:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.301852:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24401, 24401] 00020000:00200000:0.0:1775764896.301855:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99946496 chunk: [99946496, 99950592) 99950592, 4096 00020000:00200000:0.0:1775764896.301857:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.301868:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24401, 24401] 00000008:00010000:0.0:1775764896.301891:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.301893:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.301902:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301912:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99946496 99950591) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301920:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301928:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.301936:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.301944:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99946496, 99950592) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.301949:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24401 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.301963:0:18302:0:(rw26.c:858:ll_write_end()) pos 99946496, len 4096, copied 4096 00000080:00200000:0.0:1775764896.301967:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000afe3a90[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.301970:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004dd684eb 17ffffc0001001 3:0 ffff90c2128311c0 24401 lru 00000080:00200000:0.0:1775764896.301977:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006492c0ec 24401: 1< 0 - - > 2< 99946496 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 667648 - | + - + - > 5< + - + - | 0 - | 3921 - -> 00000080:00200000:0.0:1775764896.301979:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000afe3a90 00000080:00200000:0.0:1775764896.301980:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.301983:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.301990:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.301999:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302006:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302014:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.302025:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.302028:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.302030:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.302035:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.302038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.302040:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.302045:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302053:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.302061:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.302064:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.302069:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.302071:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24401, 24401] 00000080:00200000:0.0:1775764896.302081:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.302085:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.302089:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99950592 i_blocks 90112 00000080:00200000:0.0:1775764896.302092:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.302107:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99950592, count: 4096 00000080:00200000:0.0:1775764896.302130:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99950592, bytes: 4096 00000080:00200000:0.0:1775764896.302134:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.302138:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24403, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.302140:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99950592, 99954688) 0 0 00000080:00200000:0.0:1775764896.302143:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.302148:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24402, 24402] 00020000:00200000:0.0:1775764896.302151:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99950592 chunk: [99950592, 99954688) 99954688, 4096 00020000:00200000:0.0:1775764896.302153:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.302164:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24402, 24402] 00000008:00010000:0.0:1775764896.302169:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.302172:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.302179:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302188:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99950592 99954687) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302198:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302206:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302220:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.302228:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99950592, 99954688) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.302232:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24402 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.302247:0:18302:0:(rw26.c:858:ll_write_end()) pos 99950592, len 4096, copied 4096 00000080:00200000:0.0:1775764896.302251:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e5bd567c[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.302254:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d575f9ea 17ffffc0001001 3:0 ffff90c2128317e0 24402 lru 00000080:00200000:0.0:1775764896.302260:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009503fa06 24402: 1< 0 - - > 2< 99950592 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 663552 - | + - + - > 5< + - + - | 0 - | 3922 - -> 00000080:00200000:0.0:1775764896.302262:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e5bd567c 00000080:00200000:0.0:1775764896.302263:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.302265:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.302273:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302280:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302287:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302295:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.302307:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.302310:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.302312:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.302317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.302320:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.302323:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.302327:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302335:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.302342:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.302345:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.302349:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.302352:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24402, 24402] 00000080:00200000:0.0:1775764896.302362:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.302365:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.302370:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99954688 i_blocks 90112 00000080:00200000:0.0:1775764896.302373:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.302387:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99954688, count: 4096 00000080:00200000:0.0:1775764896.302398:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99954688, bytes: 4096 00000080:00200000:0.0:1775764896.302401:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.302405:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24404, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.302406:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99954688, 99958784) 0 0 00000080:00200000:0.0:1775764896.302409:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.302414:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24403, 24403] 00020000:00200000:0.0:1775764896.302416:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99954688 chunk: [99954688, 99958784) 99958784, 4096 00020000:00200000:0.0:1775764896.302419:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.302430:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24403, 24403] 00000008:00010000:0.0:1775764896.302434:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.302437:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.302445:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302453:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99954688 99958783) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302462:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302470:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302476:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.302484:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99954688, 99958784) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.302489:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24403 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.302506:0:18302:0:(rw26.c:858:ll_write_end()) pos 99954688, len 4096, copied 4096 00000080:00200000:0.0:1775764896.302510:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f155f962[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.302513:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b602606e 17ffffc0001001 3:0 ffff90c2128319a0 24403 lru 00000080:00200000:0.0:1775764896.302519:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003d9e06a1 24403: 1< 0 - - > 2< 99954688 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 659456 - | + - + - > 5< + - + - | 0 - | 3923 - -> 00000080:00200000:0.0:1775764896.302521:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f155f962 00000080:00200000:0.0:1775764896.302522:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.302524:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.302531:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302539:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302546:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302554:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.302565:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.302568:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.302570:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.302576:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.302579:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.302582:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.302585:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302593:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.302601:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.302603:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.302607:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.302609:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24403, 24403] 00000080:00200000:0.0:1775764896.302619:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.302622:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.302627:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99958784 i_blocks 90112 00000080:00200000:0.0:1775764896.302629:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.302645:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99958784, count: 4096 00000080:00200000:0.0:1775764896.302657:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99958784, bytes: 4096 00000080:00200000:0.0:1775764896.302659:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.302662:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24405, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.302664:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99958784, 99962880) 0 0 00000080:00200000:0.0:1775764896.302666:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.302671:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24404, 24404] 00020000:00200000:0.0:1775764896.302674:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99958784 chunk: [99958784, 99962880) 99962880, 4096 00020000:00200000:0.0:1775764896.302676:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.302687:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24404, 24404] 00000008:00010000:0.0:1775764896.302692:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.302695:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.302702:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302711:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99958784 99962879) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302719:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302726:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302733:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.302740:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99958784, 99962880) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.302745:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24404 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.302761:0:18302:0:(rw26.c:858:ll_write_end()) pos 99958784, len 4096, copied 4096 00000080:00200000:0.0:1775764896.302765:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f4923b7a[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.302768:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e924c3bf 17ffffc0001001 3:0 ffff90c2128312a0 24404 lru 00000080:00200000:0.0:1775764896.302774:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c3b2475d 24404: 1< 0 - - > 2< 99958784 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 655360 - | + - + - > 5< + - + - | 0 - | 3924 - -> 00000080:00200000:0.0:1775764896.302776:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f4923b7a 00000080:00200000:0.0:1775764896.302777:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.302779:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.302786:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302794:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302801:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.302808:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.302819:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.302822:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.302824:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.302830:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.302833:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.302835:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.302839:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302846:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.302854:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.302857:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.302861:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.302864:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24404, 24404] 00000080:00200000:0.0:1775764896.302892:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.302895:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.302900:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99962880 i_blocks 90112 00000080:00200000:0.0:1775764896.302904:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.302919:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99962880, count: 4096 00000080:00200000:0.0:1775764896.302931:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99962880, bytes: 4096 00000080:00200000:0.0:1775764896.302934:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.302937:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24406, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.302939:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99962880, 99966976) 0 0 00000080:00200000:0.0:1775764896.302941:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.302946:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24405, 24405] 00020000:00200000:0.0:1775764896.302949:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99962880 chunk: [99962880, 99966976) 99966976, 4096 00020000:00200000:0.0:1775764896.302951:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.302963:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24405, 24405] 00000008:00010000:0.0:1775764896.302968:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.302971:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.302978:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302985:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99962880 99966975) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302991:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.302999:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303004:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.303008:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99962880, 99966976) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.303012:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24405 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.303024:0:18302:0:(rw26.c:858:ll_write_end()) pos 99962880, len 4096, copied 4096 00000080:00200000:0.0:1775764896.303027:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ff4933c9[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.303028:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000bbe698fd 17ffffc0001001 3:0 ffff90c212831d20 24405 lru 00000080:00200000:0.0:1775764896.303032:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000be11ccfc 24405: 1< 0 - - > 2< 99962880 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 651264 - | + - + - > 5< + - + - | 0 - | 3925 - -> 00000080:00200000:0.0:1775764896.303033:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ff4933c9 00000080:00200000:0.0:1775764896.303034:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.303035:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.303040:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303045:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303049:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303053:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.303064:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.303065:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.303067:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.303071:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.303072:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.303074:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.303076:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303081:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.303086:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.303087:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.303090:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.303092:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24405, 24405] 00000080:00200000:0.0:1775764896.303099:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.303101:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.303104:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99966976 i_blocks 90112 00000080:00200000:0.0:1775764896.303106:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.303129:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99966976, count: 4096 00000080:00200000:0.0:1775764896.303138:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99966976, bytes: 4096 00000080:00200000:0.0:1775764896.303139:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.303142:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24407, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.303143:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99966976, 99971072) 0 0 00000080:00200000:0.0:1775764896.303144:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.303148:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24406, 24406] 00020000:00200000:0.0:1775764896.303150:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99966976 chunk: [99966976, 99971072) 99971072, 4096 00020000:00200000:0.0:1775764896.303152:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.303159:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24406, 24406] 00000008:00010000:0.0:1775764896.303162:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.303164:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.303169:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303175:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99966976 99971071) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303180:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303185:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303188:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.303193:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99966976, 99971072) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.303196:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24406 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.303218:0:18302:0:(rw26.c:858:ll_write_end()) pos 99966976, len 4096, copied 4096 00000080:00200000:0.0:1775764896.303221:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006e8b2dd7[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.303222:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000aa471deb 17ffffc0001001 3:0 ffff90c212832d20 24406 lru 00000080:00200000:0.0:1775764896.303226:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008019a254 24406: 1< 0 - - > 2< 99966976 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 647168 - | + - + - > 5< + - + - | 0 - | 3926 - -> 00000080:00200000:0.0:1775764896.303227:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006e8b2dd7 00000080:00200000:0.0:1775764896.303227:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.303229:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.303234:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303239:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303243:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303250:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.303261:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.303264:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.303266:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.303272:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.303274:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.303276:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.303280:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303289:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.303296:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.303299:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.303304:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.303306:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24406, 24406] 00000080:00200000:0.0:1775764896.303317:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.303320:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.303325:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99971072 i_blocks 90112 00000080:00200000:0.0:1775764896.303327:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.303343:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99971072, count: 4096 00000080:00200000:0.0:1775764896.303354:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99971072, bytes: 4096 00000080:00200000:0.0:1775764896.303357:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.303360:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24408, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.303362:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99971072, 99975168) 0 0 00000080:00200000:0.0:1775764896.303364:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.303369:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24407, 24407] 00020000:00200000:0.0:1775764896.303372:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99971072 chunk: [99971072, 99975168) 99975168, 4096 00020000:00200000:0.0:1775764896.303373:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.303384:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24407, 24407] 00000008:00010000:0.0:1775764896.303389:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.303391:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.303398:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303406:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99971072 99975167) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303414:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303421:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303427:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.303435:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99971072, 99975168) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.303441:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24407 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.303457:0:18302:0:(rw26.c:858:ll_write_end()) pos 99971072, len 4096, copied 4096 00000080:00200000:0.0:1775764896.303461:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000136b94c2[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.303464:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000070e7e384 17ffffc0001001 3:0 ffff90c212832380 24407 lru 00000080:00200000:0.0:1775764896.303469:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000229af04b 24407: 1< 0 - - > 2< 99971072 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 643072 - | + - + - > 5< + - + - | 0 - | 3927 - -> 00000080:00200000:0.0:1775764896.303471:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000136b94c2 00000080:00200000:0.0:1775764896.303472:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.303474:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.303482:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303490:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303497:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303504:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.303515:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.303518:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.303521:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.303526:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.303529:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.303531:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.303535:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303543:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.303551:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.303554:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.303558:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.303561:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24407, 24407] 00000080:00200000:0.0:1775764896.303570:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.303573:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.303578:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99975168 i_blocks 90112 00000080:00200000:0.0:1775764896.303580:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.303596:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99975168, count: 4096 00000080:00200000:0.0:1775764896.303607:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99975168, bytes: 4096 00000080:00200000:0.0:1775764896.303610:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.303613:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24409, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.303614:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99975168, 99979264) 0 0 00000080:00200000:0.0:1775764896.303617:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.303621:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24408, 24408] 00020000:00200000:0.0:1775764896.303624:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99975168 chunk: [99975168, 99979264) 99979264, 4096 00020000:00200000:0.0:1775764896.303625:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.303637:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24408, 24408] 00000008:00010000:0.0:1775764896.303642:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.303644:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.303651:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303659:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99975168 99979263) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303668:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303676:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303683:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.303691:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99975168, 99979264) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.303696:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24408 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.303711:0:18302:0:(rw26.c:858:ll_write_end()) pos 99975168, len 4096, copied 4096 00000080:00200000:0.0:1775764896.303715:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c6a06d9f[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.303718:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000390162b1 17ffffc0001001 3:0 ffff90c212832620 24408 lru 00000080:00200000:0.0:1775764896.303723:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002122f560 24408: 1< 0 - - > 2< 99975168 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 638976 - | + - + - > 5< + - + - | 0 - | 3928 - -> 00000080:00200000:0.0:1775764896.303726:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c6a06d9f 00000080:00200000:0.0:1775764896.303727:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.303729:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.303737:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303744:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303751:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.303763:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.303775:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.303778:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.303780:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.303786:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.303788:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.303790:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.303795:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303802:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.303809:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.303811:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.303816:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.303818:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24408, 24408] 00000080:00200000:0.0:1775764896.303828:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.303831:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.303836:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99979264 i_blocks 90112 00000080:00200000:0.0:1775764896.303838:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.303853:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99979264, count: 4096 00000080:00200000:0.0:1775764896.303865:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99979264, bytes: 4096 00000080:00200000:0.0:1775764896.303868:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.303891:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24410, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.303893:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99979264, 99983360) 0 0 00000080:00200000:0.0:1775764896.303895:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.303900:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24409, 24409] 00020000:00200000:0.0:1775764896.303902:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99979264 chunk: [99979264, 99983360) 99983360, 4096 00020000:00200000:0.0:1775764896.303904:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.303916:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24409, 24409] 00000008:00010000:0.0:1775764896.303921:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.303924:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.303930:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303938:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99979264 99983359) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303946:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303953:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.303960:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.303966:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99979264, 99983360) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.303971:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24409 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.303987:0:18302:0:(rw26.c:858:ll_write_end()) pos 99979264, len 4096, copied 4096 00000080:00200000:0.0:1775764896.303991:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003e932990[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.303993:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c2c002fc 17ffffc0001001 3:0 ffff90c212832e00 24409 lru 00000080:00200000:0.0:1775764896.303999:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000510391cd 24409: 1< 0 - - > 2< 99979264 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 634880 - | + - + - > 5< + - + - | 0 - | 3929 - -> 00000080:00200000:0.0:1775764896.304001:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003e932990 00000080:00200000:0.0:1775764896.304002:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.304004:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.304012:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.304019:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.304026:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.304033:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.304045:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.304047:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.304049:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.304056:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.304059:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.304061:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.304065:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304072:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.304080:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.304083:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.304087:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.304090:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24409, 24409] 00000080:00200000:0.0:1775764896.304100:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.304103:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.304107:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99983360 i_blocks 90112 00000080:00200000:0.0:1775764896.304110:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.304436:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99983360, count: 4096 00000080:00200000:0.0:1775764896.304448:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99983360, bytes: 4096 00000080:00200000:0.0:1775764896.304450:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.304454:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24411, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.304456:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99983360, 99987456) 0 0 00000080:00200000:0.0:1775764896.304459:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.304464:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24410, 24410] 00020000:00200000:0.0:1775764896.304467:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99983360 chunk: [99983360, 99987456) 99987456, 4096 00020000:00200000:0.0:1775764896.304469:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.304481:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24410, 24410] 00000008:00010000:0.0:1775764896.304486:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.304489:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.304497:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304506:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99983360 99987455) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304514:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304521:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304532:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.304539:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99983360, 99987456) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.304544:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24410 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.304561:0:18302:0:(rw26.c:858:ll_write_end()) pos 99983360, len 4096, copied 4096 00000080:00200000:0.0:1775764896.304565:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d4414506[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.304568:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000041f50be9 17ffffc0001001 3:0 ffff90c212832540 24410 lru 00000080:00200000:0.0:1775764896.304574:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000abc3ba64 24410: 1< 0 - - > 2< 99983360 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 630784 - | + - + - > 5< + - + - | 0 - | 3930 - -> 00000080:00200000:0.0:1775764896.304576:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d4414506 00000080:00200000:0.0:1775764896.304577:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.304579:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.304587:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.304595:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.304602:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.304608:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.304620:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.304622:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.304625:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.304641:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.304644:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.304646:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.304650:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304659:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.304667:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.304670:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.304675:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.304678:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24410, 24410] 00000080:00200000:0.0:1775764896.304689:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.304692:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.304729:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99987456 i_blocks 90112 00000080:00200000:0.0:1775764896.304732:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.304748:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99987456, count: 4096 00000080:00200000:0.0:1775764896.304758:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99987456, bytes: 4096 00000080:00200000:0.0:1775764896.304761:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.304765:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24412, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.304766:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99987456, 99991552) 0 0 00000080:00200000:0.0:1775764896.304768:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.304853:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24411, 24411] 00020000:00200000:0.0:1775764896.304856:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99987456 chunk: [99987456, 99991552) 99991552, 4096 00020000:00200000:0.0:1775764896.304858:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.304935:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24411, 24411] 00000008:00010000:0.0:1775764896.304942:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.304945:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.304953:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304962:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99987456 99991551) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304970:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304978:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.304985:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.304993:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99987456, 99991552) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.304999:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24411 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.305016:0:18302:0:(rw26.c:858:ll_write_end()) pos 99987456, len 4096, copied 4096 00000080:00200000:0.0:1775764896.305020:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000034b27da1[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.305023:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ff65642d 17ffffc0001001 3:0 ffff90c212832460 24411 lru 00000080:00200000:0.0:1775764896.305028:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000851570f3 24411: 1< 0 - - > 2< 99987456 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 626688 - | + - + - > 5< + - + - | 0 - | 3931 - -> 00000080:00200000:0.0:1775764896.305030:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000034b27da1 00000080:00200000:0.0:1775764896.305031:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.305033:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.305041:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.305050:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.305057:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.305064:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.305076:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.305080:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.305082:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.305093:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.305095:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.305098:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.305102:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305110:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.305130:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.305132:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.305137:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.305140:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24411, 24411] 00000080:00200000:0.0:1775764896.305151:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.305154:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.305185:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99991552 i_blocks 90112 00000080:00200000:0.0:1775764896.305189:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.305206:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99991552, count: 4096 00000080:00200000:0.0:1775764896.305215:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99991552, bytes: 4096 00000080:00200000:0.0:1775764896.305217:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.305220:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24413, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.305221:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99991552, 99995648) 0 0 00000080:00200000:0.0:1775764896.305222:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.305226:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24412, 24412] 00020000:00200000:0.0:1775764896.305228:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99991552 chunk: [99991552, 99995648) 99995648, 4096 00020000:00200000:0.0:1775764896.305230:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.305237:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24412, 24412] 00000008:00010000:0.0:1775764896.305241:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.305243:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.305250:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305259:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99991552 99995647) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305267:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305275:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305282:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.305291:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99991552, 99995648) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.305297:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24412 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.305315:0:18302:0:(rw26.c:858:ll_write_end()) pos 99991552, len 4096, copied 4096 00000080:00200000:0.0:1775764896.305320:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000096a8eb86[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.305323:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000203a07e6 17ffffc0001001 3:0 ffff90c212832a80 24412 lru 00000080:00200000:0.0:1775764896.305328:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e087f870 24412: 1< 0 - - > 2< 99991552 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 622592 - | + - + - > 5< + - + - | 0 - | 3932 - -> 00000080:00200000:0.0:1775764896.305331:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000096a8eb86 00000080:00200000:0.0:1775764896.305332:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.305334:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.305342:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.305349:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.305356:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.305363:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.305373:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.305376:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.305378:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.305387:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.305390:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.305392:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.305396:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305404:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.305413:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.305415:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.305419:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.305422:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24412, 24412] 00000080:00200000:0.0:1775764896.305432:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.305435:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.305466:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99995648 i_blocks 90112 00000080:00200000:0.0:1775764896.305470:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.305485:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99995648, count: 4096 00000080:00200000:0.0:1775764896.305496:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99995648, bytes: 4096 00000080:00200000:0.0:1775764896.305499:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.305503:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24414, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.305504:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99995648, 99999744) 0 0 00000080:00200000:0.0:1775764896.305507:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.305512:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24413, 24413] 00020000:00200000:0.0:1775764896.305515:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99995648 chunk: [99995648, 99999744) 99999744, 4096 00020000:00200000:0.0:1775764896.305517:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.305529:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24413, 24413] 00000008:00010000:0.0:1775764896.305534:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.305537:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.305545:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305554:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99995648 99999743) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305562:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305574:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305581:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.305590:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99995648, 99999744) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.305595:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24413 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.305619:0:18302:0:(rw26.c:858:ll_write_end()) pos 99995648, len 4096, copied 4096 00000080:00200000:0.0:1775764896.305624:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004558fc0f[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.305627:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000032a630f 17ffffc0001021 2:0 ffff90c212832700 24413 lru 00000080:00200000:0.0:1775764896.305632:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006ab571b9 24413: 1< 0 - - > 2< 99995648 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 618496 - | + - + - > 5< + - + - | 0 - | 3933 - -> 00000080:00200000:0.0:1775764896.305634:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004558fc0f 00000080:00200000:0.0:1775764896.305635:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.305638:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.305645:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.305652:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.305658:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.305665:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.305681:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.305684:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.305686:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.305696:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.305699:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.305701:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.305705:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.305713:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.305721:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.305724:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.305728:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.305731:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24413, 24413] 00000080:00200000:0.0:1775764896.305741:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.305745:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.305946:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 99999744 i_blocks 90112 00000080:00200000:0.0:1775764896.305950:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.305971:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 99999744, count: 4096 00000080:00200000:0.0:1775764896.305983:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 99999744, bytes: 4096 00000080:00200000:0.0:1775764896.305986:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.305990:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24415, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.305991:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [99999744, 100003840) 0 0 00000080:00200000:0.0:1775764896.305994:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.305999:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24414, 24414] 00020000:00200000:0.0:1775764896.306003:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 99999744 chunk: [99999744, 100003840) 100003840, 4096 00020000:00200000:0.0:1775764896.306005:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.306018:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24414, 24414] 00000008:00010000:0.0:1775764896.306023:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.306026:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.306034:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306042:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (99999744 100003839) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306052:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306059:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306066:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.306074:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [99999744, 100003840) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.306079:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24414 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.306096:0:18302:0:(rw26.c:858:ll_write_end()) pos 99999744, len 4096, copied 4096 00000080:00200000:0.0:1775764896.306101:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000866773e8[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.306104:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f9b7a747 17ffffc0001001 3:0 ffff90c212832c40 24414 lru 00000080:00200000:0.0:1775764896.306109:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000441a3d5 24414: 1< 0 - - > 2< 99999744 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 614400 - | + - + - > 5< + - + - | 0 - | 3934 - -> 00000080:00200000:0.0:1775764896.306123:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000866773e8 00000080:00200000:0.0:1775764896.306124:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.306127:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.306136:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.306143:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.306151:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.306158:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.306169:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.306172:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.306175:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.306185:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.306188:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.306190:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.306195:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306202:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.306210:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.306213:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.306217:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.306220:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24414, 24414] 00000080:00200000:0.0:1775764896.306230:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.306233:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.306263:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100003840 i_blocks 90112 00000080:00200000:0.0:1775764896.306267:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.306283:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100003840, count: 4096 00000080:00200000:0.0:1775764896.306293:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100003840, bytes: 4096 00000080:00200000:0.0:1775764896.306296:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.306301:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24416, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.306302:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100003840, 100007936) 0 0 00000080:00200000:0.0:1775764896.306305:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.306310:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24415, 24415] 00020000:00200000:0.0:1775764896.306313:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100003840 chunk: [100003840, 100007936) 100007936, 4096 00020000:00200000:0.0:1775764896.306316:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.306327:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24415, 24415] 00000008:00010000:0.0:1775764896.306332:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.306334:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.306342:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306351:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100003840 100007935) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306361:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306368:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306375:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.306383:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100003840, 100007936) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.306388:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24415 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.306405:0:18302:0:(rw26.c:858:ll_write_end()) pos 100003840, len 4096, copied 4096 00000080:00200000:0.0:1775764896.306409:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005b27491b[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.306412:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000025b69dcf 17ffffc0001001 3:0 ffff90c2128328c0 24415 lru 00000080:00200000:0.0:1775764896.306418:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000091832718 24415: 1< 0 - - > 2< 100003840 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 610304 - | + - + - > 5< + - + - | 0 - | 3935 - -> 00000080:00200000:0.0:1775764896.306420:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005b27491b 00000080:00200000:0.0:1775764896.306421:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.306423:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.306431:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.306438:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.306445:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.306453:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.306464:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.306467:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.306469:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.306479:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.306482:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.306485:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.306489:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306496:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.306504:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.306507:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.306512:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.306514:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24415, 24415] 00000080:00200000:0.0:1775764896.306525:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.306529:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.306557:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100007936 i_blocks 90112 00000080:00200000:0.0:1775764896.306561:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.306577:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100007936, count: 4096 00000080:00200000:0.0:1775764896.306589:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100007936, bytes: 4096 00000080:00200000:0.0:1775764896.306591:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.306595:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24417, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.306597:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100007936, 100012032) 0 0 00000080:00200000:0.0:1775764896.306600:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.306605:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24416, 24416] 00020000:00200000:0.0:1775764896.306608:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100007936 chunk: [100007936, 100012032) 100012032, 4096 00020000:00200000:0.0:1775764896.306611:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.306623:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24416, 24416] 00000008:00010000:0.0:1775764896.306628:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.306630:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.306638:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306647:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100007936 100012031) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306656:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306664:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306671:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.306679:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100007936, 100012032) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.306684:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24416 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.306701:0:18302:0:(rw26.c:858:ll_write_end()) pos 100007936, len 4096, copied 4096 00000080:00200000:0.0:1775764896.306705:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005a6c2fd9[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.306708:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000072f3d1b0 17ffffc0001001 3:0 ffff90c212832b60 24416 lru 00000080:00200000:0.0:1775764896.306714:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e7e8978b 24416: 1< 0 - - > 2< 100007936 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 606208 - | + - + - > 5< + - + - | 0 - | 3936 - -> 00000080:00200000:0.0:1775764896.306716:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005a6c2fd9 00000080:00200000:0.0:1775764896.306717:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.306719:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.306726:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.306734:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.306741:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.306753:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.306765:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.306768:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.306770:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.306894:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.306897:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.306900:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.306905:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.306914:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.306922:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.306924:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.306929:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.306931:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24416, 24416] 00000080:00200000:0.0:1775764896.306942:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.306945:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.306973:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100012032 i_blocks 90112 00000080:00200000:0.0:1775764896.306977:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.306993:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100012032, count: 4096 00000080:00200000:0.0:1775764896.307004:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100012032, bytes: 4096 00000080:00200000:0.0:1775764896.307007:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.307011:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24418, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.307013:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100012032, 100016128) 0 0 00000080:00200000:0.0:1775764896.307016:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.307021:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24417, 24417] 00020000:00200000:0.0:1775764896.307024:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100012032 chunk: [100012032, 100016128) 100016128, 4096 00020000:00200000:0.0:1775764896.307027:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.307038:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24417, 24417] 00000008:00010000:0.0:1775764896.307043:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.307046:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.307054:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307064:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100012032 100016127) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307073:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307080:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307086:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.307095:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100012032, 100016128) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.307100:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24417 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.307312:0:18302:0:(rw26.c:858:ll_write_end()) pos 100012032, len 4096, copied 4096 00000080:00200000:0.0:1775764896.307318:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b053c29a[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.307322:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000029ca6c46 17ffffc0001001 3:0 ffff90c212832000 24417 lru 00000080:00200000:0.0:1775764896.307329:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c69c6d05 24417: 1< 0 - - > 2< 100012032 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 602112 - | + - + - > 5< + - + - | 0 - | 3937 - -> 00000080:00200000:0.0:1775764896.307332:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b053c29a 00000080:00200000:0.0:1775764896.307333:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.307335:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.307344:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.307353:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.307361:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.307369:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.307381:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.307384:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.307386:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.307396:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.307399:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.307402:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.307406:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307414:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.307427:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.307430:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.307434:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.307437:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24417, 24417] 00000080:00200000:0.0:1775764896.307448:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.307451:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.307526:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100016128 i_blocks 90112 00000080:00200000:0.0:1775764896.307530:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.307549:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100016128, count: 4096 00000080:00200000:0.0:1775764896.307561:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100016128, bytes: 4096 00000080:00200000:0.0:1775764896.307563:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.307567:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24419, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.307568:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100016128, 100020224) 0 0 00000080:00200000:0.0:1775764896.307570:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.307576:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24418, 24418] 00020000:00200000:0.0:1775764896.307579:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100016128 chunk: [100016128, 100020224) 100020224, 4096 00020000:00200000:0.0:1775764896.307581:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.307593:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24418, 24418] 00000008:00010000:0.0:1775764896.307598:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.307601:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.307609:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307618:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100016128 100020223) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307626:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307638:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307646:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.307654:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100016128, 100020224) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.307659:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24418 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.307677:0:18302:0:(rw26.c:858:ll_write_end()) pos 100016128, len 4096, copied 4096 00000080:00200000:0.0:1775764896.307682:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000360692e4[3 00000000b99c1f5d 2 1 000000005c018550] 00000080:00200000:0.0:1775764896.307685:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000045041dd4 17ffffc0001001 3:0 ffff90c212832ee0 24418 lru 00000080:00200000:0.0:1775764896.307691:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005bb94d5a 24418: 1< 0 - - > 2< 100016128 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 598016 - | + - + - > 5< + - + - | 0 - | 3938 - -> 00000080:00200000:0.0:1775764896.307693:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000360692e4 00000080:00200000:0.0:1775764896.307693:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.307696:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.307704:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.307711:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 0000000009aff25c ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.307719:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.307727:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.307744:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.307747:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.307749:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.307760:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.307762:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.307764:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.307768:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.307868:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.307969:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81908/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.307971:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81908/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.307976:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.307979:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24418, 24418] 00000080:00200000:0.0:1775764896.307989:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.307992:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.308111:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100020224 i_blocks 90112 00000080:00200000:1.0:1775764896.308126:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.308149:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100020224, count: 4096 00000080:00200000:1.0:1775764896.308166:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100020224, bytes: 4096 00000080:00200000:1.0:1775764896.308169:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.308173:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24420, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.308175:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100020224, 100024320) 0 0 00000080:00200000:1.0:1775764896.308177:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.308183:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24419, 24419] 00020000:00200000:1.0:1775764896.308187:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100020224 chunk: [100020224, 100024320) 100024320, 4096 00020000:00200000:1.0:1775764896.308189:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.308205:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24419, 24419] 00000008:00010000:1.0:1775764896.308213:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.308216:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.308225:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.308234:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100020224 100024319) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.308242:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.308302:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.308315:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.308324:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100020224, 100024320) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.308330:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24419 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.308355:0:18302:0:(rw26.c:858:ll_write_end()) pos 100020224, len 4096, copied 4096 00000080:00200000:1.0:1775764896.308360:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000034fbfaa6[3 00000000b99c1f5d 2 1 000000004b0f90c8] 00000080:00200000:1.0:1775764896.308364:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000048a05b12 17ffffc0001021 2:0 ffff90c2128341c0 24419 lru 00000080:00200000:1.0:1775764896.308370:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d3d3fdbe 24419: 1< 0 - - > 2< 100020224 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 593920 - | + - + - > 5< + - + - | 0 - | 3939 - -> 00000080:00200000:1.0:1775764896.308373:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000034fbfaa6 00000080:00200000:1.0:1775764896.308374:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.308376:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.308384:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ee1f743f ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.308392:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ee1f743f ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.308400:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.308407:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.308420:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.308422:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.308425:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.308438:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea038 (ffff90c2195eef38/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.308441:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea038 (ffff90c2195eef38/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.308444:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.308448:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.308456:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.308464:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea038 (ffff90c2195eef38/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.308466:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195eef38/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.308471:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.308474:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24419, 24419] 00000080:00200000:1.0:1775764896.308629:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.308633:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.308745:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100024320 i_blocks 90112 00000080:00200000:0.0:1775764896.308750:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.308861:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100024320, count: 4096 00000080:00200000:0.0:1775764896.308943:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100024320, bytes: 4096 00000080:00200000:0.0:1775764896.308947:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.308951:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24421, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.308953:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100024320, 100028416) 0 0 00000080:00200000:0.0:1775764896.308955:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.308961:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24420, 24420] 00020000:00200000:0.0:1775764896.308964:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100024320 chunk: [100024320, 100028416) 100028416, 4096 00020000:00200000:0.0:1775764896.308967:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.308983:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24420, 24420] 00000008:00010000:0.0:1775764896.308989:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.308992:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.309000:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309010:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100024320 100028415) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309019:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309027:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309034:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.309042:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100024320, 100028416) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.309048:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24420 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.309068:0:18302:0:(rw26.c:858:ll_write_end()) pos 100024320, len 4096, copied 4096 00000080:00200000:0.0:1775764896.309073:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001e0c2ab1[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.309076:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005dbd143a 17ffffc0001001 3:0 ffff90c2128320e0 24420 lru 00000080:00200000:0.0:1775764896.309082:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002f515332 24420: 1< 0 - - > 2< 100024320 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 589824 - | + - + - > 5< + - + - | 0 - | 3940 - -> 00000080:00200000:0.0:1775764896.309084:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001e0c2ab1 00000080:00200000:0.0:1775764896.309086:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.309088:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.309098:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.309107:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.309126:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.309134:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.309146:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.309149:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.309152:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.309164:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.309166:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.309168:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.309172:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309180:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.309189:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.309191:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.309196:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.309199:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24420, 24420] 00000080:00200000:0.0:1775764896.309212:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.309216:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.309312:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100028416 i_blocks 90112 00000080:00200000:0.0:1775764896.309316:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.309335:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100028416, count: 4096 00000080:00200000:0.0:1775764896.309346:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100028416, bytes: 4096 00000080:00200000:0.0:1775764896.309349:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.309352:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24422, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.309354:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100028416, 100032512) 0 0 00000080:00200000:0.0:1775764896.309357:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.309362:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24421, 24421] 00020000:00200000:0.0:1775764896.309365:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100028416 chunk: [100028416, 100032512) 100032512, 4096 00020000:00200000:0.0:1775764896.309367:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.309379:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24421, 24421] 00000008:00010000:0.0:1775764896.309384:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.309386:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.309394:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309403:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100028416 100032511) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309411:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309418:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309425:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.309433:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100028416, 100032512) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.309440:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24421 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.309458:0:18302:0:(rw26.c:858:ll_write_end()) pos 100028416, len 4096, copied 4096 00000080:00200000:0.0:1775764896.309462:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000586cacb0[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.309465:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a7b20b9f 17ffffc0001001 3:0 ffff90c2128321c0 24421 lru 00000080:00200000:0.0:1775764896.309471:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000cdc111c8 24421: 1< 0 - - > 2< 100028416 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 585728 - | + - + - > 5< + - + - | 0 - | 3941 - -> 00000080:00200000:0.0:1775764896.309473:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000586cacb0 00000080:00200000:0.0:1775764896.309474:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.309476:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.309484:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.309491:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.309499:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.309507:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.309520:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.309523:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.309525:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.309536:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.309539:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.309541:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.309545:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309553:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.309561:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.309563:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.309567:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.309569:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24421, 24421] 00000080:00200000:0.0:1775764896.309580:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.309583:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.309614:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100032512 i_blocks 90112 00000080:00200000:0.0:1775764896.309617:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.309633:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100032512, count: 4096 00000080:00200000:0.0:1775764896.309643:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100032512, bytes: 4096 00000080:00200000:0.0:1775764896.309646:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.309650:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24423, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.309652:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100032512, 100036608) 0 0 00000080:00200000:0.0:1775764896.309654:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.309660:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24422, 24422] 00020000:00200000:0.0:1775764896.309663:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100032512 chunk: [100032512, 100036608) 100036608, 4096 00020000:00200000:0.0:1775764896.309665:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.309676:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24422, 24422] 00000008:00010000:0.0:1775764896.309681:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.309684:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.309691:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309701:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100032512 100036607) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309709:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309715:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.309722:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.309729:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100032512, 100036608) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.309735:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24422 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.309752:0:18302:0:(rw26.c:858:ll_write_end()) pos 100032512, len 4096, copied 4096 00000080:00200000:0.0:1775764896.309756:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000134f9e92[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.309759:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f6449398 17ffffc0001001 3:0 ffff90c2128327e0 24422 lru 00000080:00200000:0.0:1775764896.309765:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f9a6c265 24422: 1< 0 - - > 2< 100032512 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 581632 - | + - + - > 5< + - + - | 0 - | 3942 - -> 00000080:00200000:0.0:1775764896.309767:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000134f9e92 00000080:00200000:0.0:1775764896.309768:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.309867:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.309955:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.309963:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.309970:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.309983:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.309997:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.310000:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.310003:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.310014:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.310016:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.310019:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.310023:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310031:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.310039:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.310042:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.310046:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.310049:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24422, 24422] 00000080:00200000:0.0:1775764896.310061:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.310064:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.310096:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100036608 i_blocks 90112 00000080:00200000:0.0:1775764896.310100:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.310129:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100036608, count: 4096 00000080:00200000:0.0:1775764896.310142:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100036608, bytes: 4096 00000080:00200000:0.0:1775764896.310145:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.310148:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24424, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.310150:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100036608, 100040704) 0 0 00000080:00200000:0.0:1775764896.310152:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.310157:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24423, 24423] 00020000:00200000:0.0:1775764896.310160:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100036608 chunk: [100036608, 100040704) 100040704, 4096 00020000:00200000:0.0:1775764896.310163:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.310173:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24423, 24423] 00000008:00010000:0.0:1775764896.310178:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.310181:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.310188:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310196:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100036608 100040703) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310204:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310211:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310218:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.310226:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100036608, 100040704) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.310231:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24423 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.310299:0:18302:0:(rw26.c:858:ll_write_end()) pos 100036608, len 4096, copied 4096 00000080:00200000:0.0:1775764896.310304:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cd70cd9c[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.310307:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d3ee49f1 17ffffc0001001 3:0 ffff90c2128329a0 24423 lru 00000080:00200000:0.0:1775764896.310313:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f2571a38 24423: 1< 0 - - > 2< 100036608 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 577536 - | + - + - > 5< + - + - | 0 - | 3943 - -> 00000080:00200000:0.0:1775764896.310315:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cd70cd9c 00000080:00200000:0.0:1775764896.310317:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.310319:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.310328:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.310335:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.310341:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.310347:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.310358:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.310360:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.310362:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.310372:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.310375:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.310377:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.310381:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310389:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.310401:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.310404:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.310408:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.310411:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24423, 24423] 00000080:00200000:0.0:1775764896.310421:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.310425:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.310454:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100040704 i_blocks 90112 00000080:00200000:0.0:1775764896.310457:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.310472:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100040704, count: 4096 00000080:00200000:0.0:1775764896.310482:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100040704, bytes: 4096 00000080:00200000:0.0:1775764896.310486:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.310490:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24425, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.310491:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100040704, 100044800) 0 0 00000080:00200000:0.0:1775764896.310493:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.310499:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24424, 24424] 00020000:00200000:0.0:1775764896.310501:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100040704 chunk: [100040704, 100044800) 100044800, 4096 00020000:00200000:0.0:1775764896.310503:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.310515:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24424, 24424] 00000008:00010000:0.0:1775764896.310521:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.310523:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.310531:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310539:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100040704 100044799) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310548:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310560:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310567:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.310574:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100040704, 100044800) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.310580:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24424 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.310596:0:18302:0:(rw26.c:858:ll_write_end()) pos 100040704, len 4096, copied 4096 00000080:00200000:0.0:1775764896.310600:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000de506ff0[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.310603:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000093edec14 17ffffc0001001 3:0 ffff90c2128322a0 24424 lru 00000080:00200000:0.0:1775764896.310609:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000dcda2076 24424: 1< 0 - - > 2< 100040704 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 573440 - | + - + - > 5< + - + - | 0 - | 3944 - -> 00000080:00200000:0.0:1775764896.310611:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000de506ff0 00000080:00200000:0.0:1775764896.310612:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.310614:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.310622:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.310630:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.310637:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.310643:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.310659:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.310663:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.310665:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.310675:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.310677:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.310681:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.310685:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.310692:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.310699:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.310701:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.310706:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.310709:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24424, 24424] 00000080:00200000:0.0:1775764896.310720:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.310723:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.311036:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100044800 i_blocks 90112 00000080:00200000:0.0:1775764896.311041:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.311058:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100044800, count: 4096 00000080:00200000:0.0:1775764896.311068:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100044800, bytes: 4096 00000080:00200000:0.0:1775764896.311070:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.311075:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24426, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.311077:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100044800, 100048896) 0 0 00000080:00200000:0.0:1775764896.311079:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.311086:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24425, 24425] 00020000:00200000:0.0:1775764896.311089:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100044800 chunk: [100044800, 100048896) 100048896, 4096 00020000:00200000:0.0:1775764896.311092:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.311103:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24425, 24425] 00000008:00010000:0.0:1775764896.311109:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.311124:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.311134:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311143:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100044800 100048895) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311152:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311160:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311167:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.311175:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100044800, 100048896) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.311181:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24425 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.311210:0:18302:0:(rw26.c:858:ll_write_end()) pos 100044800, len 4096, copied 4096 00000080:00200000:0.0:1775764896.311215:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000604a4aa9[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.311218:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000040975ca6 17ffffc0001001 3:0 ffff90c2128339a0 24425 lru 00000080:00200000:0.0:1775764896.311224:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c4cd05e2 24425: 1< 0 - - > 2< 100044800 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 569344 - | + - + - > 5< + - + - | 0 - | 3945 - -> 00000080:00200000:0.0:1775764896.311226:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000604a4aa9 00000080:00200000:0.0:1775764896.311227:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.311230:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.311239:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.311315:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.311322:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.311329:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.311343:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.311347:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.311349:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.311362:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.311364:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.311367:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.311370:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311378:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.311386:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.311388:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.311392:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.311395:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24425, 24425] 00000080:00200000:0.0:1775764896.311407:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.311410:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.311440:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100048896 i_blocks 90112 00000080:00200000:0.0:1775764896.311444:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.311459:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100048896, count: 4096 00000080:00200000:0.0:1775764896.311469:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100048896, bytes: 4096 00000080:00200000:0.0:1775764896.311472:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.311475:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24427, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.311477:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100048896, 100052992) 0 0 00000080:00200000:0.0:1775764896.311479:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.311485:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24426, 24426] 00020000:00200000:0.0:1775764896.311488:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100048896 chunk: [100048896, 100052992) 100052992, 4096 00020000:00200000:0.0:1775764896.311490:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.311501:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24426, 24426] 00000008:00010000:0.0:1775764896.311506:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.311508:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.311515:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311524:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100048896 100052991) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311533:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311540:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311547:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.311555:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100048896, 100052992) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.311561:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24426 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.311578:0:18302:0:(rw26.c:858:ll_write_end()) pos 100048896, len 4096, copied 4096 00000080:00200000:0.0:1775764896.311582:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000080e1eefd[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.311585:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d51e0119 17ffffc0001001 3:0 ffff90c2128332a0 24426 lru 00000080:00200000:0.0:1775764896.311590:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007dda7771 24426: 1< 0 - - > 2< 100048896 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 565248 - | + - + - > 5< + - + - | 0 - | 3946 - -> 00000080:00200000:0.0:1775764896.311592:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000080e1eefd 00000080:00200000:0.0:1775764896.311594:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.311596:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.311604:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.311612:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.311619:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.311626:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.311637:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.311640:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.311642:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.311653:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.311656:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.311658:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.311662:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.311671:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.311679:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.311681:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.311686:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.311688:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24426, 24426] 00000080:00200000:0.0:1775764896.311700:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.311703:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.311732:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100052992 i_blocks 90112 00000080:00200000:0.0:1775764896.311736:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.311753:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100052992, count: 4096 00000080:00200000:0.0:1775764896.311764:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100052992, bytes: 4096 00000080:00200000:0.0:1775764896.311767:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.311863:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24428, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.311865:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100052992, 100057088) 0 0 00000080:00200000:0.0:1775764896.311869:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.311967:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24427, 24427] 00020000:00200000:0.0:1775764896.311971:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100052992 chunk: [100052992, 100057088) 100057088, 4096 00020000:00200000:0.0:1775764896.311973:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.311989:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24427, 24427] 00000008:00010000:0.0:1775764896.311996:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.311998:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.312007:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312017:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100052992 100057087) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312026:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312033:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312040:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.312048:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100052992, 100057088) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.312054:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24427 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.312072:0:18302:0:(rw26.c:858:ll_write_end()) pos 100052992, len 4096, copied 4096 00000080:00200000:0.0:1775764896.312076:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000aa8564db[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.312079:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002de917f9 17ffffc0001001 3:0 ffff90c212833d20 24427 lru 00000080:00200000:0.0:1775764896.312085:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ebd28239 24427: 1< 0 - - > 2< 100052992 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 561152 - | + - + - > 5< + - + - | 0 - | 3947 - -> 00000080:00200000:0.0:1775764896.312087:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000aa8564db 00000080:00200000:0.0:1775764896.312088:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.312091:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.312099:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.312108:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.312129:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.312142:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.312155:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.312158:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.312160:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.312171:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.312173:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.312176:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.312180:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312188:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.312196:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.312199:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.312203:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.312207:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24427, 24427] 00000080:00200000:0.0:1775764896.312219:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.312221:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.312311:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100057088 i_blocks 90112 00000080:00200000:0.0:1775764896.312315:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.312331:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100057088, count: 4096 00000080:00200000:0.0:1775764896.312343:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100057088, bytes: 4096 00000080:00200000:0.0:1775764896.312346:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.312349:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24429, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.312351:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100057088, 100061184) 0 0 00000080:00200000:0.0:1775764896.312353:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.312359:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24428, 24428] 00020000:00200000:0.0:1775764896.312362:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100057088 chunk: [100057088, 100061184) 100061184, 4096 00020000:00200000:0.0:1775764896.312364:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.312376:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24428, 24428] 00000008:00010000:0.0:1775764896.312382:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.312384:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.312392:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312402:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100057088 100061183) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312410:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312418:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312426:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.312434:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100057088, 100061184) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.312439:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24428 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.312458:0:18302:0:(rw26.c:858:ll_write_end()) pos 100057088, len 4096, copied 4096 00000080:00200000:0.0:1775764896.312462:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007ffc28fd[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.312465:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000382c5974 17ffffc0001001 3:0 ffff90c212833380 24428 lru 00000080:00200000:0.0:1775764896.312472:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000072e00c6f 24428: 1< 0 - - > 2< 100057088 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 557056 - | + - + - > 5< + - + - | 0 - | 3948 - -> 00000080:00200000:0.0:1775764896.312474:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007ffc28fd 00000080:00200000:0.0:1775764896.312475:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.312478:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.312485:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.312494:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.312501:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.312508:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.312520:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.312523:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.312525:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.312536:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.312539:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.312541:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.312545:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312553:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.312566:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.312568:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.312572:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.312575:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24428, 24428] 00000080:00200000:0.0:1775764896.312586:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.312589:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.312620:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100061184 i_blocks 90112 00000080:00200000:0.0:1775764896.312625:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.312640:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100061184, count: 4096 00000080:00200000:0.0:1775764896.312651:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100061184, bytes: 4096 00000080:00200000:0.0:1775764896.312653:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.312658:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24430, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.312659:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100061184, 100065280) 0 0 00000080:00200000:0.0:1775764896.312662:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.312667:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24429, 24429] 00020000:00200000:0.0:1775764896.312670:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100061184 chunk: [100061184, 100065280) 100065280, 4096 00020000:00200000:0.0:1775764896.312672:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.312683:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24429, 24429] 00000008:00010000:0.0:1775764896.312689:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.312691:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.312699:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312708:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100061184 100065279) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312717:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312730:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.312737:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.312745:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100061184, 100065280) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.312750:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24429 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.313151:0:18302:0:(rw26.c:858:ll_write_end()) pos 100061184, len 4096, copied 4096 00000080:00200000:0.0:1775764896.313157:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000007196c75[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.313160:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000067808b5d 17ffffc0001021 2:0 ffff90c212833620 24429 lru 00000080:00200000:0.0:1775764896.313167:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003cf59f14 24429: 1< 0 - - > 2< 100061184 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 552960 - | + - + - > 5< + - + - | 0 - | 3949 - -> 00000080:00200000:0.0:1775764896.313169:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000007196c75 00000080:00200000:0.0:1775764896.313170:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.313172:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.313182:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.313190:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.313198:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.313206:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.313224:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.313227:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.313229:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.313240:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.313243:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.313320:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.313325:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.313333:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.313343:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.313346:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.313351:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.313354:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24429, 24429] 00000080:00200000:0.0:1775764896.313366:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.313370:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.313403:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100065280 i_blocks 90112 00000080:00200000:0.0:1775764896.313407:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.313423:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100065280, count: 4096 00000080:00200000:0.0:1775764896.313435:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100065280, bytes: 4096 00000080:00200000:0.0:1775764896.313438:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.313442:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24431, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.313444:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100065280, 100069376) 0 0 00000080:00200000:0.0:1775764896.313446:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.313453:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24430, 24430] 00020000:00200000:0.0:1775764896.313457:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100065280 chunk: [100065280, 100069376) 100069376, 4096 00020000:00200000:0.0:1775764896.313460:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.313473:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24430, 24430] 00000008:00010000:0.0:1775764896.313539:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.313542:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.313552:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.313562:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100065280 100069375) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.313570:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.313578:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.313585:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.313594:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100065280, 100069376) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.313601:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24430 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.313622:0:18302:0:(rw26.c:858:ll_write_end()) pos 100065280, len 4096, copied 4096 00000080:00200000:0.0:1775764896.313627:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005adaf4e6[3 00000000b99c1f5d 2 1 0000000082e3e0b6] 00000080:00200000:0.0:1775764896.313630:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c611da0e 17ffffc0001001 3:0 ffff90c212833e00 24430 lru 00000080:00200000:0.0:1775764896.313635:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000069460abf 24430: 1< 0 - - > 2< 100065280 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 548864 - | + - + - > 5< + - + - | 0 - | 3950 - -> 00000080:00200000:0.0:1775764896.313638:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005adaf4e6 00000080:00200000:0.0:1775764896.313639:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.313641:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.313650:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.313658:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000ff2d9a0b ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.313665:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.313673:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.313685:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.313688:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.313690:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.313701:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.313703:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.313705:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.313709:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.313717:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.313725:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1308 (ffff90c210c81d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.313728:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.313732:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.313735:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24430, 24430] 00000080:00200000:0.0:1775764896.313747:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.313750:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.313938:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100069376 i_blocks 90112 00000080:00200000:1.0:1775764896.313942:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.313963:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100069376, count: 4096 00000080:00200000:1.0:1775764896.313980:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100069376, bytes: 4096 00000080:00200000:1.0:1775764896.313983:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.313986:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24432, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.313988:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100069376, 100073472) 0 0 00000080:00200000:1.0:1775764896.313990:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.313996:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24431, 24431] 00020000:00200000:1.0:1775764896.313998:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100069376 chunk: [100069376, 100073472) 100073472, 4096 00020000:00200000:1.0:1775764896.314000:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.314016:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24431, 24431] 00000008:00010000:1.0:1775764896.314023:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.314027:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.314035:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314044:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100069376 100073471) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314053:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314074:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314081:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.314090:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100069376, 100073472) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.314096:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24431 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.314122:0:18302:0:(rw26.c:858:ll_write_end()) pos 100069376, len 4096, copied 4096 00000080:00200000:1.0:1775764896.314127:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004a974567[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.314130:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000da996f4b 17ffffc0001001 3:0 ffff90c2128347e0 24431 lru 00000080:00200000:1.0:1775764896.314135:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000fc4f515f 24431: 1< 0 - - > 2< 100069376 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 544768 - | + - + - > 5< + - + - | 0 - | 3951 - -> 00000080:00200000:1.0:1775764896.314137:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004a974567 00000080:00200000:1.0:1775764896.314138:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.314141:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.314149:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.314158:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.314165:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.314172:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.314183:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.314186:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.314189:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.314196:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.314199:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.314202:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.314206:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314214:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.314223:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.314225:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.314229:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.314232:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24431, 24431] 00000080:00200000:1.0:1775764896.314261:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.314265:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.314270:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100073472 i_blocks 90112 00000080:00200000:1.0:1775764896.314273:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.314291:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100073472, count: 4096 00000080:00200000:1.0:1775764896.314303:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100073472, bytes: 4096 00000080:00200000:1.0:1775764896.314306:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.314309:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24433, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.314310:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100073472, 100077568) 0 0 00000080:00200000:1.0:1775764896.314313:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.314318:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24432, 24432] 00020000:00200000:1.0:1775764896.314320:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100073472 chunk: [100073472, 100077568) 100077568, 4096 00020000:00200000:1.0:1775764896.314322:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.314332:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24432, 24432] 00000008:00010000:1.0:1775764896.314337:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.314340:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.314347:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314356:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100073472 100077567) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314365:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314371:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314377:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.314386:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100073472, 100077568) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.314392:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24432 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.314409:0:18302:0:(rw26.c:858:ll_write_end()) pos 100073472, len 4096, copied 4096 00000080:00200000:1.0:1775764896.314414:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000013bd4367[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.314417:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c052d1ed 17ffffc0001001 3:0 ffff90c2128349a0 24432 lru 00000080:00200000:1.0:1775764896.314423:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000018e52e7 24432: 1< 0 - - > 2< 100073472 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 540672 - | + - + - > 5< + - + - | 0 - | 3952 - -> 00000080:00200000:1.0:1775764896.314425:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000013bd4367 00000080:00200000:1.0:1775764896.314426:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.314428:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.314436:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.314443:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.314450:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.314458:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.314470:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.314473:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.314486:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.314494:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.314496:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.314499:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.314502:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314509:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.314517:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.314519:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.314523:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.314526:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24432, 24432] 00000080:00200000:1.0:1775764896.314536:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.314539:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.314544:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100077568 i_blocks 90112 00000080:00200000:1.0:1775764896.314547:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.314561:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100077568, count: 4096 00000080:00200000:1.0:1775764896.314572:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100077568, bytes: 4096 00000080:00200000:1.0:1775764896.314574:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.314578:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24434, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.314580:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100077568, 100081664) 0 0 00000080:00200000:1.0:1775764896.314582:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.314587:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24433, 24433] 00020000:00200000:1.0:1775764896.314590:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100077568 chunk: [100077568, 100081664) 100081664, 4096 00020000:00200000:1.0:1775764896.314592:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.314604:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24433, 24433] 00000008:00010000:1.0:1775764896.314609:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.314611:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.314619:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314627:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100077568 100081663) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314636:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314649:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314655:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.314662:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100077568, 100081664) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.314668:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24433 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.314684:0:18302:0:(rw26.c:858:ll_write_end()) pos 100077568, len 4096, copied 4096 00000080:00200000:1.0:1775764896.314688:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005921f745[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.314691:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007653f1da 17ffffc0001001 3:0 ffff90c2128342a0 24433 lru 00000080:00200000:1.0:1775764896.314696:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000371c99b7 24433: 1< 0 - - > 2< 100077568 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 536576 - | + - + - > 5< + - + - | 0 - | 3953 - -> 00000080:00200000:1.0:1775764896.314698:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005921f745 00000080:00200000:1.0:1775764896.314699:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.314701:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.314709:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.314717:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.314724:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.314732:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.314749:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.314752:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.314754:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.314760:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.314763:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.314765:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.314769:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314790:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.314798:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.314801:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.314805:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.314808:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24433, 24433] 00000080:00200000:1.0:1775764896.314818:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.314821:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.314826:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100081664 i_blocks 90112 00000080:00200000:1.0:1775764896.314828:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.314843:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100081664, count: 4096 00000080:00200000:1.0:1775764896.314854:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100081664, bytes: 4096 00000080:00200000:1.0:1775764896.314856:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.314861:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24435, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.314862:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100081664, 100085760) 0 0 00000080:00200000:1.0:1775764896.314865:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.314890:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24434, 24434] 00020000:00200000:1.0:1775764896.314894:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100081664 chunk: [100081664, 100085760) 100085760, 4096 00020000:00200000:1.0:1775764896.314897:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.314909:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24434, 24434] 00000008:00010000:1.0:1775764896.314914:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.314916:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.314924:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314933:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100081664 100085759) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314941:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314949:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.314955:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.314962:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100081664, 100085760) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.314968:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24434 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.314984:0:18302:0:(rw26.c:858:ll_write_end()) pos 100081664, len 4096, copied 4096 00000080:00200000:1.0:1775764896.314989:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004bbb334b[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.314992:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000acc2a86 17ffffc0001001 3:0 ffff90c212834d20 24434 lru 00000080:00200000:1.0:1775764896.314998:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b0aa4f56 24434: 1< 0 - - > 2< 100081664 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 532480 - | + - + - > 5< + - + - | 0 - | 3954 - -> 00000080:00200000:1.0:1775764896.315000:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004bbb334b 00000080:00200000:1.0:1775764896.315001:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.315003:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.315011:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315019:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315026:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315034:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.315044:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.315047:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.315049:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.315055:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.315057:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.315060:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.315064:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315072:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.315080:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.315083:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.315087:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.315089:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24434, 24434] 00000080:00200000:1.0:1775764896.315100:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.315104:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.315109:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100085760 i_blocks 90112 00000080:00200000:1.0:1775764896.315123:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.315141:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100085760, count: 4096 00000080:00200000:1.0:1775764896.315152:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100085760, bytes: 4096 00000080:00200000:1.0:1775764896.315155:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.315158:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24436, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.315160:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100085760, 100089856) 0 0 00000080:00200000:1.0:1775764896.315162:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.315168:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24435, 24435] 00020000:00200000:1.0:1775764896.315171:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100085760 chunk: [100085760, 100089856) 100089856, 4096 00020000:00200000:1.0:1775764896.315173:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.315185:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24435, 24435] 00000008:00010000:1.0:1775764896.315190:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.315192:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.315200:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315210:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100085760 100089855) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315218:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315226:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315234:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.315243:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100085760, 100089856) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.315248:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24435 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.315265:0:18302:0:(rw26.c:858:ll_write_end()) pos 100085760, len 4096, copied 4096 00000080:00200000:1.0:1775764896.315269:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f3026f54[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.315272:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000026541dbf 17ffffc0001001 3:0 ffff90c212834380 24435 lru 00000080:00200000:1.0:1775764896.315278:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000092cb9d18 24435: 1< 0 - - > 2< 100085760 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 528384 - | + - + - > 5< + - + - | 0 - | 3955 - -> 00000080:00200000:1.0:1775764896.315280:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f3026f54 00000080:00200000:1.0:1775764896.315281:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.315283:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.315292:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315300:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315307:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315315:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.315326:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.315330:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.315332:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.315338:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.315340:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.315343:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.315347:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315355:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.315364:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.315366:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.315370:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.315373:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24435, 24435] 00000080:00200000:1.0:1775764896.315384:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.315387:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.315392:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100089856 i_blocks 90112 00000080:00200000:1.0:1775764896.315395:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.315409:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100089856, count: 4096 00000080:00200000:1.0:1775764896.315420:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100089856, bytes: 4096 00000080:00200000:1.0:1775764896.315423:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.315426:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24437, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.315428:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100089856, 100093952) 0 0 00000080:00200000:1.0:1775764896.315430:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.315435:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24436, 24436] 00020000:00200000:1.0:1775764896.315439:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100089856 chunk: [100089856, 100093952) 100093952, 4096 00020000:00200000:1.0:1775764896.315441:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.315452:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24436, 24436] 00000008:00010000:1.0:1775764896.315457:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.315460:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.315467:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315476:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100089856 100093951) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315485:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315493:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315499:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.315507:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100089856, 100093952) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.315512:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24436 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.315531:0:18302:0:(rw26.c:858:ll_write_end()) pos 100089856, len 4096, copied 4096 00000080:00200000:1.0:1775764896.315536:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c10be189[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.315539:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004f8d010c 17ffffc0001001 3:0 ffff90c212834620 24436 lru 00000080:00200000:1.0:1775764896.315545:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ad5012ff 24436: 1< 0 - - > 2< 100089856 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 524288 - | + - + - > 5< + - + - | 0 - | 3956 - -> 00000080:00200000:1.0:1775764896.315547:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c10be189 00000080:00200000:1.0:1775764896.315548:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.315550:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.315558:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315566:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315572:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315585:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.315597:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.315600:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.315602:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.315608:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.315611:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.315613:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.315617:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315624:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.315634:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.315636:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.315641:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.315644:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24436, 24436] 00000080:00200000:1.0:1775764896.315655:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.315658:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.315663:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100093952 i_blocks 90112 00000080:00200000:1.0:1775764896.315666:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.315681:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100093952, count: 4096 00000080:00200000:1.0:1775764896.315693:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100093952, bytes: 4096 00000080:00200000:1.0:1775764896.315695:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.315699:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24438, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.315701:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100093952, 100098048) 0 0 00000080:00200000:1.0:1775764896.315703:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.315708:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24437, 24437] 00020000:00200000:1.0:1775764896.315711:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100093952 chunk: [100093952, 100098048) 100098048, 4096 00020000:00200000:1.0:1775764896.315713:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.315724:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24437, 24437] 00000008:00010000:1.0:1775764896.315730:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.315732:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.315740:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315749:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100093952 100098047) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315758:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315765:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315772:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.315779:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100093952, 100098048) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.315784:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24437 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.315810:0:18302:0:(rw26.c:858:ll_write_end()) pos 100093952, len 4096, copied 4096 00000080:00200000:1.0:1775764896.315815:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000fc84ff93[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.315818:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000227774c8 17ffffc0001001 3:0 ffff90c212835a80 24437 lru 00000080:00200000:1.0:1775764896.315824:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000bcdf2dce 24437: 1< 0 - - > 2< 100093952 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 520192 - | + - + - > 5< + - + - | 0 - | 3957 - -> 00000080:00200000:1.0:1775764896.315827:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000fc84ff93 00000080:00200000:1.0:1775764896.315828:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.315830:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.315838:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315847:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315854:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.315862:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.315898:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.315901:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.315903:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.315909:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.315911:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.315913:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.315917:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.315924:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.315937:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.315939:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.315943:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.315946:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24437, 24437] 00000080:00200000:1.0:1775764896.315957:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.315960:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.315964:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100098048 i_blocks 90112 00000080:00200000:1.0:1775764896.315967:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.315982:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100098048, count: 4096 00000080:00200000:1.0:1775764896.315993:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100098048, bytes: 4096 00000080:00200000:1.0:1775764896.315995:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.315999:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24439, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.316001:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100098048, 100102144) 0 0 00000080:00200000:1.0:1775764896.316003:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.316008:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24438, 24438] 00020000:00200000:1.0:1775764896.316011:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100098048 chunk: [100098048, 100102144) 100102144, 4096 00020000:00200000:1.0:1775764896.316013:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.316023:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24438, 24438] 00000008:00010000:1.0:1775764896.316028:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.316030:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.316037:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316045:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100098048 100102143) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316054:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316066:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316073:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.316081:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100098048, 100102144) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.316085:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24438 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.316101:0:18302:0:(rw26.c:858:ll_write_end()) pos 100098048, len 4096, copied 4096 00000080:00200000:1.0:1775764896.316105:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003fce4d15[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.316108:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000687af2b6 17ffffc0001001 3:0 ffff90c212835700 24438 lru 00000080:00200000:1.0:1775764896.316127:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c28a47ce 24438: 1< 0 - - > 2< 100098048 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 516096 - | + - + - > 5< + - + - | 0 - | 3958 - -> 00000080:00200000:1.0:1775764896.316130:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003fce4d15 00000080:00200000:1.0:1775764896.316131:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.316133:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.316141:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316148:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316155:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316162:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.316178:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.316181:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.316184:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.316189:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.316192:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.316194:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.316198:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316205:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.316213:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.316215:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.316218:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.316220:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24438, 24438] 00000080:00200000:1.0:1775764896.316231:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.316233:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.316238:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100102144 i_blocks 90112 00000080:00200000:1.0:1775764896.316240:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.316254:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100102144, count: 4096 00000080:00200000:1.0:1775764896.316265:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100102144, bytes: 4096 00000080:00200000:1.0:1775764896.316268:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.316272:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24440, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.316273:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100102144, 100106240) 0 0 00000080:00200000:1.0:1775764896.316276:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.316281:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24439, 24439] 00020000:00200000:1.0:1775764896.316284:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100102144 chunk: [100102144, 100106240) 100106240, 4096 00020000:00200000:1.0:1775764896.316286:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.316297:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24439, 24439] 00000008:00010000:1.0:1775764896.316302:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.316304:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.316311:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316320:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100102144 100106239) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316328:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316336:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316343:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.316351:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100102144, 100106240) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.316356:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24439 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.316373:0:18302:0:(rw26.c:858:ll_write_end()) pos 100102144, len 4096, copied 4096 00000080:00200000:1.0:1775764896.316377:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000069b8b4b5[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.316379:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000893d3e4a 17ffffc0001001 3:0 ffff90c212835c40 24439 lru 00000080:00200000:1.0:1775764896.316384:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000497dc31f 24439: 1< 0 - - > 2< 100102144 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 512000 - | + - + - > 5< + - + - | 0 - | 3959 - -> 00000080:00200000:1.0:1775764896.316387:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000069b8b4b5 00000080:00200000:1.0:1775764896.316388:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.316390:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.316397:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316405:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316411:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316418:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.316428:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.316430:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.316432:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.316437:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.316439:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.316442:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.316445:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316452:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.316458:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.316460:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.316464:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.316466:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24439, 24439] 00000080:00200000:1.0:1775764896.316476:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.316479:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.316484:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100106240 i_blocks 90112 00000080:00200000:1.0:1775764896.316487:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.316502:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100106240, count: 4096 00000080:00200000:1.0:1775764896.316513:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100106240, bytes: 4096 00000080:00200000:1.0:1775764896.316515:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.316519:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24441, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.316521:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100106240, 100110336) 0 0 00000080:00200000:1.0:1775764896.316523:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.316528:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24440, 24440] 00020000:00200000:1.0:1775764896.316531:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100106240 chunk: [100106240, 100110336) 100110336, 4096 00020000:00200000:1.0:1775764896.316533:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.316544:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24440, 24440] 00000008:00010000:1.0:1775764896.316549:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.316551:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.316558:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316566:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100106240 100110335) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316574:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316582:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316589:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.316598:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100106240, 100110336) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.316603:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24440 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.316619:0:18302:0:(rw26.c:858:ll_write_end()) pos 100106240, len 4096, copied 4096 00000080:00200000:1.0:1775764896.316623:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a5090a27[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.316625:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007997aa5f 17ffffc0001001 3:0 ffff90c2128358c0 24440 lru 00000080:00200000:1.0:1775764896.316631:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000080868664 24440: 1< 0 - - > 2< 100106240 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 507904 - | + - + - > 5< + - + - | 0 - | 3960 - -> 00000080:00200000:1.0:1775764896.316633:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a5090a27 00000080:00200000:1.0:1775764896.316634:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.316636:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.316643:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316651:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316657:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316665:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.316676:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.316680:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.316682:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.316688:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.316690:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.316692:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.316697:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316704:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.316712:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.316713:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.316717:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.316720:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24440, 24440] 00000080:00200000:1.0:1775764896.316730:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.316733:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.316737:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100110336 i_blocks 90112 00000080:00200000:1.0:1775764896.316740:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.316754:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100110336, count: 4096 00000080:00200000:1.0:1775764896.316765:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100110336, bytes: 4096 00000080:00200000:1.0:1775764896.316768:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.316771:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24442, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.316773:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100110336, 100114432) 0 0 00000080:00200000:1.0:1775764896.316775:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.316780:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24441, 24441] 00020000:00200000:1.0:1775764896.316783:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100110336 chunk: [100110336, 100114432) 100114432, 4096 00020000:00200000:1.0:1775764896.316785:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.316796:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24441, 24441] 00000008:00010000:1.0:1775764896.316801:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.316804:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.316811:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316819:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100110336 100114431) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316828:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316836:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316843:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.316850:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100110336, 100114432) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.316857:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24441 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.316898:0:18302:0:(rw26.c:858:ll_write_end()) pos 100110336, len 4096, copied 4096 00000080:00200000:1.0:1775764896.316903:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d98a686b[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.316906:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cce865eb 17ffffc0001001 3:0 ffff90c212835b60 24441 lru 00000080:00200000:1.0:1775764896.316911:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006c23c4de 24441: 1< 0 - - > 2< 100110336 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 503808 - | + - + - > 5< + - + - | 0 - | 3961 - -> 00000080:00200000:1.0:1775764896.316913:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d98a686b 00000080:00200000:1.0:1775764896.316914:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.316916:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.316924:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316931:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316937:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.316948:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.316959:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.316962:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.316965:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.316970:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.316973:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.316975:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.316979:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.316986:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.316993:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.316996:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.317000:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.317002:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24441, 24441] 00000080:00200000:1.0:1775764896.317012:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.317015:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.317020:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100114432 i_blocks 90112 00000080:00200000:1.0:1775764896.317022:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.317035:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100114432, count: 4096 00000080:00200000:1.0:1775764896.317046:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100114432, bytes: 4096 00000080:00200000:1.0:1775764896.317049:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.317053:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24443, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.317054:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100114432, 100118528) 0 0 00000080:00200000:1.0:1775764896.317056:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.317061:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24442, 24442] 00020000:00200000:1.0:1775764896.317064:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100114432 chunk: [100114432, 100118528) 100118528, 4096 00020000:00200000:1.0:1775764896.317066:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.317076:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24442, 24442] 00000008:00010000:1.0:1775764896.317081:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.317083:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.317090:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.317099:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100114432 100118527) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.317106:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318167:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318180:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.318191:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100114432, 100118528) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.318201:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24442 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.318228:0:18302:0:(rw26.c:858:ll_write_end()) pos 100114432, len 4096, copied 4096 00000080:00200000:1.0:1775764896.318232:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f617bda8[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.318234:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000069df7518 17ffffc0001001 3:0 ffff90c212835000 24442 lru 00000080:00200000:1.0:1775764896.318239:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000093c7e0b5 24442: 1< 0 - - > 2< 100114432 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 499712 - | + - + - > 5< + - + - | 0 - | 3962 - -> 00000080:00200000:1.0:1775764896.318240:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f617bda8 00000080:00200000:1.0:1775764896.318241:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.318318:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.318330:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.318336:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.318341:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.318346:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.318356:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.318358:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.318360:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.318376:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.318378:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.318379:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.318382:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318388:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.318400:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.318401:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.318406:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.318409:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24442, 24442] 00000080:00200000:1.0:1775764896.318421:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.318424:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.318453:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100118528 i_blocks 90112 00000080:00200000:1.0:1775764896.318456:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.318626:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100118528, count: 4096 00000080:00200000:1.0:1775764896.318643:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100118528, bytes: 4096 00000080:00200000:1.0:1775764896.318646:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.318651:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24444, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.318653:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100118528, 100122624) 0 0 00000080:00200000:1.0:1775764896.318656:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.318665:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24443, 24443] 00020000:00200000:1.0:1775764896.318670:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100118528 chunk: [100118528, 100122624) 100122624, 4096 00020000:00200000:1.0:1775764896.318683:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.318698:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24443, 24443] 00000008:00010000:1.0:1775764896.318704:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.318706:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.318715:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318723:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100118528 100122623) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318729:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318738:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318742:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.318746:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100118528, 100122624) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.318750:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24443 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.318763:0:18302:0:(rw26.c:858:ll_write_end()) pos 100118528, len 4096, copied 4096 00000080:00200000:1.0:1775764896.318766:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009b27b691[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.318768:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e801b5ad 17ffffc0001001 3:0 ffff90c212835ee0 24443 lru 00000080:00200000:1.0:1775764896.318773:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002dfb75ba 24443: 1< 0 - - > 2< 100118528 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 495616 - | + - + - > 5< + - + - | 0 - | 3963 - -> 00000080:00200000:1.0:1775764896.318774:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009b27b691 00000080:00200000:1.0:1775764896.318775:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.318777:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.318785:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.318792:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.318799:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.318806:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.318822:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.318825:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.318828:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.318838:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.318841:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.318843:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.318847:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318855:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.318861:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.318862:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.318865:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.318867:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24443, 24443] 00000080:00200000:1.0:1775764896.318890:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.318892:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.318912:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100122624 i_blocks 90112 00000080:00200000:1.0:1775764896.318915:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.318926:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100122624, count: 4096 00000080:00200000:1.0:1775764896.318932:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100122624, bytes: 4096 00000080:00200000:1.0:1775764896.318934:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.318936:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24445, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.318937:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100122624, 100126720) 0 0 00000080:00200000:1.0:1775764896.318939:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.318943:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24444, 24444] 00020000:00200000:1.0:1775764896.318945:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100122624 chunk: [100122624, 100126720) 100126720, 4096 00020000:00200000:1.0:1775764896.318947:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.318953:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24444, 24444] 00000008:00010000:1.0:1775764896.318957:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.318958:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.318964:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318970:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100122624 100126719) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318975:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318980:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.318984:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.318989:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100122624, 100126720) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.318992:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24444 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.319007:0:18302:0:(rw26.c:858:ll_write_end()) pos 100122624, len 4096, copied 4096 00000080:00200000:1.0:1775764896.319009:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000077fbd13f[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.319011:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000786bf3ff 17ffffc0001001 3:0 ffff90c2128350e0 24444 lru 00000080:00200000:1.0:1775764896.319015:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000059e51468 24444: 1< 0 - - > 2< 100122624 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 491520 - | + - + - > 5< + - + - | 0 - | 3964 - -> 00000080:00200000:1.0:1775764896.319017:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000077fbd13f 00000080:00200000:1.0:1775764896.319018:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.319019:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.319024:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319029:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319034:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319038:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.319044:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.319046:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.319047:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.319054:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.319055:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.319057:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.319059:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319063:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.319069:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.319070:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.319073:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.319075:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24444, 24444] 00000080:00200000:1.0:1775764896.319082:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.319084:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.319103:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100126720 i_blocks 90112 00000080:00200000:1.0:1775764896.319105:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.319130:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100126720, count: 4096 00000080:00200000:1.0:1775764896.319141:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100126720, bytes: 4096 00000080:00200000:1.0:1775764896.319144:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.319147:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24446, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.319149:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100126720, 100130816) 0 0 00000080:00200000:1.0:1775764896.319151:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.319156:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24445, 24445] 00020000:00200000:1.0:1775764896.319159:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100126720 chunk: [100126720, 100130816) 100130816, 4096 00020000:00200000:1.0:1775764896.319161:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.319172:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24445, 24445] 00000008:00010000:1.0:1775764896.319177:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.319179:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.319187:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319196:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100126720 100130815) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319203:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319208:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319213:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.319218:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100126720, 100130816) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.319222:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24445 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.319241:0:18302:0:(rw26.c:858:ll_write_end()) pos 100126720, len 4096, copied 4096 00000080:00200000:1.0:1775764896.319320:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000193e54d9[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.319323:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000527906ff 17ffffc0001021 2:0 ffff90c2128351c0 24445 lru 00000080:00200000:1.0:1775764896.319327:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000534c3980 24445: 1< 0 - - > 2< 100126720 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 487424 - | + - + - > 5< + - + - | 0 - | 3965 - -> 00000080:00200000:1.0:1775764896.319328:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000193e54d9 00000080:00200000:1.0:1775764896.319329:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.319331:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.319338:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319343:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319347:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319351:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.319359:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.319361:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.319362:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.319369:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.319371:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.319372:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.319375:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319380:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.319386:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.319387:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.319390:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.319392:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24445, 24445] 00000080:00200000:1.0:1775764896.319399:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.319401:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.319421:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100130816 i_blocks 90112 00000080:00200000:1.0:1775764896.319423:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.319434:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100130816, count: 4096 00000080:00200000:1.0:1775764896.319440:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100130816, bytes: 4096 00000080:00200000:1.0:1775764896.319442:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.319444:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24447, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.319446:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100130816, 100134912) 0 0 00000080:00200000:1.0:1775764896.319447:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.319451:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24446, 24446] 00020000:00200000:1.0:1775764896.319453:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100130816 chunk: [100130816, 100134912) 100134912, 4096 00020000:00200000:1.0:1775764896.319455:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.319461:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24446, 24446] 00000008:00010000:1.0:1775764896.319465:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.319467:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.319473:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319480:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100130816 100134911) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319489:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319497:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319503:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.319511:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100130816, 100134912) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.319516:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24446 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.319532:0:18302:0:(rw26.c:858:ll_write_end()) pos 100130816, len 4096, copied 4096 00000080:00200000:1.0:1775764896.319594:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006a99f9cd[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.319597:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b424eb29 17ffffc0001001 3:0 ffff90c2128357e0 24446 lru 00000080:00200000:1.0:1775764896.319601:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000928bb823 24446: 1< 0 - - > 2< 100130816 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 483328 - | + - + - > 5< + - + - | 0 - | 3966 - -> 00000080:00200000:1.0:1775764896.319602:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006a99f9cd 00000080:00200000:1.0:1775764896.319603:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.319605:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.319612:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319617:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319621:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319630:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.319637:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.319639:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.319640:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.319647:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.319649:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.319651:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.319653:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319658:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.319663:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.319665:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.319667:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.319669:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24446, 24446] 00000080:00200000:1.0:1775764896.319680:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.319683:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.319712:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100134912 i_blocks 90112 00000080:00200000:1.0:1775764896.319716:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.319732:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100134912, count: 4096 00000080:00200000:1.0:1775764896.319740:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100134912, bytes: 4096 00000080:00200000:1.0:1775764896.319742:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.319745:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24448, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.319746:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100134912, 100139008) 0 0 00000080:00200000:1.0:1775764896.319748:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.319752:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24447, 24447] 00020000:00200000:1.0:1775764896.319754:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100134912 chunk: [100134912, 100139008) 100139008, 4096 00020000:00200000:1.0:1775764896.319756:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.319763:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24447, 24447] 00000008:00010000:1.0:1775764896.319766:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.319768:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.319775:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319784:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100134912 100139007) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319792:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319799:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319805:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.319813:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100134912, 100139008) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.319818:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24447 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.319836:0:18302:0:(rw26.c:858:ll_write_end()) pos 100134912, len 4096, copied 4096 00000080:00200000:1.0:1775764896.319840:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000ffba51a[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.319844:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008b5546f4 17ffffc0001001 3:0 ffff90c2128359a0 24447 lru 00000080:00200000:1.0:1775764896.319849:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000315a1b30 24447: 1< 0 - - > 2< 100134912 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 479232 - | + - + - > 5< + - + - | 0 - | 3967 - -> 00000080:00200000:1.0:1775764896.319852:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000ffba51a 00000080:00200000:1.0:1775764896.319853:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.319855:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.319862:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319867:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319900:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.319908:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.319919:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.319922:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.319924:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.319933:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.319936:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.319938:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.319942:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.319949:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.319958:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.319960:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.319963:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.319965:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24447, 24447] 00000080:00200000:1.0:1775764896.319973:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.319975:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.319995:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100139008 i_blocks 90112 00000080:00200000:1.0:1775764896.319998:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.320008:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100139008, count: 4096 00000080:00200000:1.0:1775764896.320014:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100139008, bytes: 4096 00000080:00200000:1.0:1775764896.320016:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.320018:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24449, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.320020:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100139008, 100143104) 0 0 00000080:00200000:1.0:1775764896.320021:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.320025:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24448, 24448] 00020000:00200000:1.0:1775764896.320027:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100139008 chunk: [100139008, 100143104) 100143104, 4096 00020000:00200000:1.0:1775764896.320028:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.320035:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24448, 24448] 00000008:00010000:1.0:1775764896.320038:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.320040:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.320045:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320051:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100139008 100143103) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320057:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320065:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320069:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.320073:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100139008, 100143104) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.320077:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24448 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.320109:0:18302:0:(rw26.c:858:ll_write_end()) pos 100139008, len 4096, copied 4096 00000080:00200000:1.0:1775764896.320126:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000af6db48a[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.320128:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fd05d7e1 17ffffc0001001 3:0 ffff90c2128352a0 24448 lru 00000080:00200000:1.0:1775764896.320132:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000087dce087 24448: 1< 0 - - > 2< 100139008 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 475136 - | + - + - > 5< + - + - | 0 - | 3968 - -> 00000080:00200000:1.0:1775764896.320133:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000af6db48a 00000080:00200000:1.0:1775764896.320133:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.320135:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.320141:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.320146:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.320150:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.320154:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.320165:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.320167:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.320169:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.320176:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.320177:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.320179:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.320182:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320187:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.320192:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.320194:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.320196:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.320197:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24448, 24448] 00000080:00200000:1.0:1775764896.320205:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.320207:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.320226:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100143104 i_blocks 90112 00000080:00200000:1.0:1775764896.320228:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.320239:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100143104, count: 4096 00000080:00200000:1.0:1775764896.320351:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100143104, bytes: 4096 00000080:00200000:1.0:1775764896.320355:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.320359:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24450, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.320360:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100143104, 100147200) 0 0 00000080:00200000:1.0:1775764896.320362:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.320367:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24449, 24449] 00020000:00200000:1.0:1775764896.320371:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100143104 chunk: [100143104, 100147200) 100147200, 4096 00020000:00200000:1.0:1775764896.320372:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.320384:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24449, 24449] 00000008:00010000:1.0:1775764896.320389:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.320391:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.320399:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320408:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100143104 100147199) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320416:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320424:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320431:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.320438:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100143104, 100147200) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.320442:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24449 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.320455:0:18302:0:(rw26.c:858:ll_write_end()) pos 100143104, len 4096, copied 4096 00000080:00200000:1.0:1775764896.320458:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e2ab7bc3[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.320459:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d49097d9 17ffffc0001001 3:0 ffff90c212835d20 24449 lru 00000080:00200000:1.0:1775764896.320463:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000064be8198 24449: 1< 0 - - > 2< 100143104 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 471040 - | + - + - > 5< + - + - | 0 - | 3969 - -> 00000080:00200000:1.0:1775764896.320464:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e2ab7bc3 00000080:00200000:1.0:1775764896.320465:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.320467:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.320472:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.320479:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.320486:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.320494:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.320505:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.320508:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.320510:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.320520:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.320523:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.320525:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.320529:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320626:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.320634:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.320636:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.320640:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.320642:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24449, 24449] 00000080:00200000:1.0:1775764896.320650:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.320652:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.320674:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100147200 i_blocks 90112 00000080:00200000:1.0:1775764896.320678:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.320693:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100147200, count: 4096 00000080:00200000:1.0:1775764896.320703:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100147200, bytes: 4096 00000080:00200000:1.0:1775764896.320706:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.320709:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24451, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.320711:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100147200, 100151296) 0 0 00000080:00200000:1.0:1775764896.320714:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.320719:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24450, 24450] 00020000:00200000:1.0:1775764896.320722:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100147200 chunk: [100147200, 100151296) 100151296, 4096 00020000:00200000:1.0:1775764896.320724:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.320735:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24450, 24450] 00000008:00010000:1.0:1775764896.320741:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.320743:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.320750:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320756:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100147200 100151295) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320762:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320767:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320771:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.320778:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100147200, 100151296) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.320783:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24450 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.320802:0:18302:0:(rw26.c:858:ll_write_end()) pos 100147200, len 4096, copied 4096 00000080:00200000:1.0:1775764896.320806:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c7f65fc7[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.320809:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000015bdbbd7 17ffffc0001001 3:0 ffff90c212835380 24450 lru 00000080:00200000:1.0:1775764896.320814:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000081a0455c 24450: 1< 0 - - > 2< 100147200 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 466944 - | + - + - > 5< + - + - | 0 - | 3970 - -> 00000080:00200000:1.0:1775764896.320816:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c7f65fc7 00000080:00200000:1.0:1775764896.320817:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.320819:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.320827:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.320834:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.320841:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.320849:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.320861:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.320863:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.320865:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.320894:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.320897:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.320899:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.320904:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.320911:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.320920:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.320922:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.320927:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.320929:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24450, 24450] 00000080:00200000:1.0:1775764896.320937:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.320939:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.320959:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100151296 i_blocks 90112 00000080:00200000:1.0:1775764896.320961:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.320972:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100151296, count: 4096 00000080:00200000:1.0:1775764896.320978:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100151296, bytes: 4096 00000080:00200000:1.0:1775764896.320980:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.320982:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24452, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.320983:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100151296, 100155392) 0 0 00000080:00200000:1.0:1775764896.320985:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.320989:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24451, 24451] 00020000:00200000:1.0:1775764896.320991:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100151296 chunk: [100151296, 100155392) 100155392, 4096 00020000:00200000:1.0:1775764896.320992:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.320999:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24451, 24451] 00000008:00010000:1.0:1775764896.321002:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.321004:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.321010:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321016:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100151296 100155391) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321022:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321026:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321030:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.321035:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100151296, 100155392) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.321038:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24451 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.321053:0:18302:0:(rw26.c:858:ll_write_end()) pos 100151296, len 4096, copied 4096 00000080:00200000:1.0:1775764896.321056:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003c570fea[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.321058:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f4d495d7 17ffffc0001001 3:0 ffff90c212835620 24451 lru 00000080:00200000:1.0:1775764896.321062:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000fbd7d6d 24451: 1< 0 - - > 2< 100151296 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 462848 - | + - + - > 5< + - + - | 0 - | 3971 - -> 00000080:00200000:1.0:1775764896.321063:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003c570fea 00000080:00200000:1.0:1775764896.321064:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.321065:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.321070:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.321076:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.321080:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.321088:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.321095:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.321096:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.321098:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.321104:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.321105:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.321107:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.321110:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321129:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.321138:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.321140:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.321144:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.321146:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24451, 24451] 00000080:00200000:1.0:1775764896.321154:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.321156:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.321175:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100155392 i_blocks 90112 00000080:00200000:1.0:1775764896.321178:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.321188:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100155392, count: 4096 00000080:00200000:1.0:1775764896.321194:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100155392, bytes: 4096 00000080:00200000:1.0:1775764896.321196:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.321199:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24453, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.321200:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100155392, 100159488) 0 0 00000080:00200000:1.0:1775764896.321202:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.321205:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24452, 24452] 00020000:00200000:1.0:1775764896.321207:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100155392 chunk: [100155392, 100159488) 100159488, 4096 00020000:00200000:1.0:1775764896.321209:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.321215:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24452, 24452] 00000008:00010000:1.0:1775764896.321219:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.321220:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.321226:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321231:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100155392 100159487) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321237:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321242:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321422:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.321429:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100155392, 100159488) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.321433:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24452 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.321446:0:18302:0:(rw26.c:858:ll_write_end()) pos 100155392, len 4096, copied 4096 00000080:00200000:1.0:1775764896.321449:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001aa35f6b[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.321451:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005696d55e 17ffffc0001001 3:0 ffff90c212835e00 24452 lru 00000080:00200000:1.0:1775764896.321455:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c73798f5 24452: 1< 0 - - > 2< 100155392 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 458752 - | + - + - > 5< + - + - | 0 - | 3972 - -> 00000080:00200000:1.0:1775764896.321456:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001aa35f6b 00000080:00200000:1.0:1775764896.321457:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.321459:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.321464:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.321469:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.321473:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.321479:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.321489:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.321492:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.321494:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.321504:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.321507:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.321509:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.321513:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321521:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.321532:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.321534:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.321598:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.321601:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24452, 24452] 00000080:00200000:1.0:1775764896.321609:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.321611:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.321632:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100159488 i_blocks 90112 00000080:00200000:1.0:1775764896.321635:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.321646:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100159488, count: 4096 00000080:00200000:1.0:1775764896.321653:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100159488, bytes: 4096 00000080:00200000:1.0:1775764896.321654:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.321657:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24454, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.321658:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100159488, 100163584) 0 0 00000080:00200000:1.0:1775764896.321660:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.321664:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24453, 24453] 00020000:00200000:1.0:1775764896.321666:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100159488 chunk: [100159488, 100163584) 100163584, 4096 00020000:00200000:1.0:1775764896.321667:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.321676:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24453, 24453] 00000008:00010000:1.0:1775764896.321681:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.321683:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.321691:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321699:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100159488 100163583) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321707:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321721:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.321729:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.321737:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100159488, 100163584) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.321743:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24453 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.321758:0:18302:0:(rw26.c:858:ll_write_end()) pos 100159488, len 4096, copied 4096 00000080:00200000:1.0:1775764896.321761:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d5500f47[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.321763:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000194bae46 17ffffc0001001 3:0 ffff90c212835540 24453 lru 00000080:00200000:1.0:1775764896.321767:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007aad531a 24453: 1< 0 - - > 2< 100159488 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 454656 - | + - + - > 5< + - + - | 0 - | 3973 - -> 00000080:00200000:1.0:1775764896.321768:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d5500f47 00000080:00200000:1.0:1775764896.321769:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.321770:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.321778:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.321955:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.321962:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.321969:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.321987:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.321990:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.321992:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.322000:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.322002:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.322004:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.322006:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.322012:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.322017:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.322018:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.322021:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.322023:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24453, 24453] 00000080:00200000:1.0:1775764896.322031:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.322033:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.322053:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100163584 i_blocks 90112 00000080:00200000:1.0:1775764896.322055:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.322067:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100163584, count: 4096 00000080:00200000:1.0:1775764896.322073:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100163584, bytes: 4096 00000080:00200000:1.0:1775764896.322075:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.322077:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24455, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.322078:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100163584, 100167680) 0 0 00000080:00200000:1.0:1775764896.322080:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.322083:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24454, 24454] 00020000:00200000:1.0:1775764896.322086:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100163584 chunk: [100163584, 100167680) 100167680, 4096 00020000:00200000:1.0:1775764896.322088:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.322094:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24454, 24454] 00000008:00010000:1.0:1775764896.322098:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.322099:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.322105:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.322111:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100163584 100167679) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.322129:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.322134:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.322138:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.322143:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100163584, 100167680) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.322147:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24454 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.322957:0:18302:0:(rw26.c:858:ll_write_end()) pos 100163584, len 4096, copied 4096 00000080:00200000:1.0:1775764896.322965:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c713b102[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.322967:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000060ec1847 17ffffc0001001 3:0 ffff90c212835460 24454 lru 00000080:00200000:1.0:1775764896.322972:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000373c783c 24454: 1< 0 - - > 2< 100163584 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 450560 - | + - + - > 5< + - + - | 0 - | 3974 - -> 00000080:00200000:1.0:1775764896.322973:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c713b102 00000080:00200000:1.0:1775764896.322974:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.322976:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.322990:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.322996:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323001:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323005:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.323016:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.323018:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.323019:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.323030:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.323032:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.323034:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.323037:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323042:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.323049:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.323050:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.323053:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.323056:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24454, 24454] 00000080:00200000:1.0:1775764896.323071:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.323073:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.323106:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100167680 i_blocks 90112 00000080:00200000:1.0:1775764896.323109:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.323137:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100167680, count: 4096 00000080:00200000:1.0:1775764896.323146:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100167680, bytes: 4096 00000080:00200000:1.0:1775764896.323148:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.323151:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24456, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.323152:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100167680, 100171776) 0 0 00000080:00200000:1.0:1775764896.323154:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.323160:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24455, 24455] 00020000:00200000:1.0:1775764896.323163:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100167680 chunk: [100167680, 100171776) 100171776, 4096 00020000:00200000:1.0:1775764896.323165:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.323174:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24455, 24455] 00000008:00010000:1.0:1775764896.323178:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.323181:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.323188:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323195:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100167680 100171775) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323201:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323205:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323209:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.323215:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100167680, 100171776) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.323220:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24455 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.323309:0:18302:0:(rw26.c:858:ll_write_end()) pos 100167680, len 4096, copied 4096 00000080:00200000:1.0:1775764896.323312:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001f85b26d[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.323314:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000dee4c32b 17ffffc0001001 3:0 ffff90c212836c40 24455 lru 00000080:00200000:1.0:1775764896.323318:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000054d3224 24455: 1< 0 - - > 2< 100167680 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 446464 - | + - + - > 5< + - + - | 0 - | 3975 - -> 00000080:00200000:1.0:1775764896.323320:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001f85b26d 00000080:00200000:1.0:1775764896.323321:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.323322:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.323328:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323333:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323338:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323342:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.323349:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.323351:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.323352:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.323360:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.323361:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.323362:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.323365:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323370:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.323376:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.323377:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.323380:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.323382:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24455, 24455] 00000080:00200000:1.0:1775764896.323389:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.323391:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.323411:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100171776 i_blocks 90112 00000080:00200000:1.0:1775764896.323413:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.323424:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100171776, count: 4096 00000080:00200000:1.0:1775764896.323430:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100171776, bytes: 4096 00000080:00200000:1.0:1775764896.323432:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.323434:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24457, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.323435:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100171776, 100175872) 0 0 00000080:00200000:1.0:1775764896.323437:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.323441:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24456, 24456] 00020000:00200000:1.0:1775764896.323443:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100171776 chunk: [100171776, 100175872) 100175872, 4096 00020000:00200000:1.0:1775764896.323445:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.323451:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24456, 24456] 00000008:00010000:1.0:1775764896.323455:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.323456:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.323462:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323468:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100171776 100175871) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323474:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323481:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323488:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.323495:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100171776, 100175872) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.323500:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24456 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.323518:0:18302:0:(rw26.c:858:ll_write_end()) pos 100171776, len 4096, copied 4096 00000080:00200000:1.0:1775764896.323522:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000058fb779d[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.323524:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009d5bad37 17ffffc0001001 3:0 ffff90c2128368c0 24456 lru 00000080:00200000:1.0:1775764896.323529:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000614c1dc5 24456: 1< 0 - - > 2< 100171776 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 442368 - | + - + - > 5< + - + - | 0 - | 3976 - -> 00000080:00200000:1.0:1775764896.323531:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000058fb779d 00000080:00200000:1.0:1775764896.323532:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.323534:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.323608:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323614:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323618:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323627:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.323636:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.323637:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.323639:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.323646:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.323647:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.323649:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.323652:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323657:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.323662:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.323663:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.323666:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.323667:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24456, 24456] 00000080:00200000:1.0:1775764896.323677:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.323680:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.323708:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100175872 i_blocks 90112 00000080:00200000:1.0:1775764896.323712:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.323728:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100175872, count: 4096 00000080:00200000:1.0:1775764896.323738:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100175872, bytes: 4096 00000080:00200000:1.0:1775764896.323741:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.323745:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24458, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.323746:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100175872, 100179968) 0 0 00000080:00200000:1.0:1775764896.323748:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.323752:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24457, 24457] 00020000:00200000:1.0:1775764896.323754:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100175872 chunk: [100175872, 100179968) 100179968, 4096 00020000:00200000:1.0:1775764896.323755:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.323763:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24457, 24457] 00000008:00010000:1.0:1775764896.323766:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.323768:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.323774:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323783:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100175872 100179967) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323791:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323798:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323805:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.323812:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100175872, 100179968) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.323817:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24457 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.323833:0:18302:0:(rw26.c:858:ll_write_end()) pos 100175872, len 4096, copied 4096 00000080:00200000:1.0:1775764896.323837:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e4b5ef90[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.323840:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cd3f4ec7 17ffffc0001001 3:0 ffff90c212836b60 24457 lru 00000080:00200000:1.0:1775764896.323846:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000007edb854 24457: 1< 0 - - > 2< 100175872 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 438272 - | + - + - > 5< + - + - | 0 - | 3977 - -> 00000080:00200000:1.0:1775764896.323848:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e4b5ef90 00000080:00200000:1.0:1775764896.323849:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.323851:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.323858:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323864:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323893:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.323902:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.323913:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.323917:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.323919:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.323928:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.323930:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.323932:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.323936:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.323944:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.323955:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.323956:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.323959:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.323961:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24457, 24457] 00000080:00200000:1.0:1775764896.323969:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.323971:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.323991:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100179968 i_blocks 90112 00000080:00200000:1.0:1775764896.323993:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.324004:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100179968, count: 4096 00000080:00200000:1.0:1775764896.324009:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100179968, bytes: 4096 00000080:00200000:1.0:1775764896.324011:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.324014:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24459, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.324015:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100179968, 100184064) 0 0 00000080:00200000:1.0:1775764896.324016:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.324020:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24458, 24458] 00020000:00200000:1.0:1775764896.324022:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100179968 chunk: [100179968, 100184064) 100184064, 4096 00020000:00200000:1.0:1775764896.324023:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.324030:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24458, 24458] 00000008:00010000:1.0:1775764896.324033:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.324035:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.324040:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324046:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100179968 100184063) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324051:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324060:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324064:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.324068:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100179968, 100184064) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.324071:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24458 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.324084:0:18302:0:(rw26.c:858:ll_write_end()) pos 100179968, len 4096, copied 4096 00000080:00200000:1.0:1775764896.324086:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c34739e2[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.324088:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000db00131 17ffffc0001001 3:0 ffff90c212836000 24458 lru 00000080:00200000:1.0:1775764896.324092:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a6c72572 24458: 1< 0 - - > 2< 100179968 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 434176 - | + - + - > 5< + - + - | 0 - | 3978 - -> 00000080:00200000:1.0:1775764896.324093:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c34739e2 00000080:00200000:1.0:1775764896.324093:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.324095:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.324100:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.324105:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.324109:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.324125:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.324136:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.324138:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.324139:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.324146:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.324148:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.324150:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.324152:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324157:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.324162:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.324164:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.324166:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.324168:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24458, 24458] 00000080:00200000:1.0:1775764896.324175:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.324177:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.324195:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100184064 i_blocks 90112 00000080:00200000:1.0:1775764896.324198:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.324208:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100184064, count: 4096 00000080:00200000:1.0:1775764896.324214:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100184064, bytes: 4096 00000080:00200000:1.0:1775764896.324216:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.324219:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24460, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.324220:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100184064, 100188160) 0 0 00000080:00200000:1.0:1775764896.324221:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.324225:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24459, 24459] 00020000:00200000:1.0:1775764896.324227:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100184064 chunk: [100184064, 100188160) 100188160, 4096 00020000:00200000:1.0:1775764896.324228:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.324234:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24459, 24459] 00000008:00010000:1.0:1775764896.324237:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.324239:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.324373:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324381:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100184064 100188159) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324386:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324391:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324396:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.324401:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100184064, 100188160) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.324405:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24459 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.324418:0:18302:0:(rw26.c:858:ll_write_end()) pos 100184064, len 4096, copied 4096 00000080:00200000:1.0:1775764896.324421:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005e4b442e[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.324422:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003625bdd7 17ffffc0001001 3:0 ffff90c212836ee0 24459 lru 00000080:00200000:1.0:1775764896.324426:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000fa8e3b6d 24459: 1< 0 - - > 2< 100184064 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 430080 - | + - + - > 5< + - + - | 0 - | 3979 - -> 00000080:00200000:1.0:1775764896.324427:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005e4b442e 00000080:00200000:1.0:1775764896.324427:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.324429:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.324435:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.324439:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.324444:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.324448:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.324454:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.324456:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.324457:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.324465:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.324466:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.324468:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.324470:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324474:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.324480:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.324481:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.324484:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.324486:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24459, 24459] 00000080:00200000:1.0:1775764896.324494:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.324496:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.324514:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100188160 i_blocks 90112 00000080:00200000:1.0:1775764896.324517:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.324528:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100188160, count: 4096 00000080:00200000:1.0:1775764896.324534:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100188160, bytes: 4096 00000080:00200000:1.0:1775764896.324596:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.324600:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24461, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.324601:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100188160, 100192256) 0 0 00000080:00200000:1.0:1775764896.324602:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.324607:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24460, 24460] 00020000:00200000:1.0:1775764896.324609:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100188160 chunk: [100188160, 100192256) 100192256, 4096 00020000:00200000:1.0:1775764896.324611:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.324620:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24460, 24460] 00000008:00010000:1.0:1775764896.324623:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.324625:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.324631:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324637:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100188160 100192255) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324642:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324647:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324651:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.324656:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100188160, 100192256) append 0 append lockpos 0 00000080:00200000:1.0:1775764896.324659:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24460 of 0 to 4096 bytes 00000080:00200000:1.0:1775764896.324679:0:18302:0:(rw26.c:858:ll_write_end()) pos 100188160, len 4096, copied 4096 00000080:00200000:1.0:1775764896.324684:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000033c24a30[3 00000000b99c1f5d 2 1 0000000094f4462a] 00000080:00200000:1.0:1775764896.324687:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000934ed36a 17ffffc0001021 2:0 ffff90c2128360e0 24460 lru 00000080:00200000:1.0:1775764896.324692:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000076608dcf 24460: 1< 0 - - > 2< 100188160 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 425984 - | + - + - > 5< + - + - | 0 - | 3980 - -> 00000080:00200000:1.0:1775764896.324694:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000033c24a30 00000080:00200000:1.0:1775764896.324695:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:1.0:1775764896.324697:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:1.0:1775764896.324705:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.324712:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000e53b02c9 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.324719:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:1.0:1775764896.324727:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:1.0:1775764896.324738:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:1.0:1775764896.324741:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:1.0:1775764896.324744:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:1.0:1775764896.324751:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:1.0:1775764896.324753:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:1.0:1775764896.324755:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:1.0:1775764896.324758:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:1.0:1775764896.324763:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:1.0:1775764896.324768:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:1.0:1775764896.324770:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:1.0:1775764896.324773:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:1.0:1775764896.324775:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24460, 24460] 00000080:00200000:1.0:1775764896.324786:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1775764896.324789:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1775764896.324818:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100192256 i_blocks 90112 00000080:00200000:1.0:1775764896.324821:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1775764896.324837:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100192256, count: 4096 00000080:00200000:1.0:1775764896.324848:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100192256, bytes: 4096 00000080:00200000:1.0:1775764896.324850:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:1.0:1775764896.324854:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24462, [0x200000403:0x10:0x0] hash] 00000020:00200000:1.0:1775764896.324855:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100192256, 100196352) 0 0 00000080:00200000:1.0:1775764896.324858:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1775764896.324862:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24461, 24461] 00020000:00200000:1.0:1775764896.324864:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100192256 chunk: [100192256, 100196352) 100196352, 4096 00020000:00200000:1.0:1775764896.324865:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1775764896.324900:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24461, 24461] 00000008:00010000:1.0:1775764896.324905:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:1.0:1775764896.324907:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2095b8430, osc lock ffff90c20a667300, flags 0x0 00010000:00010000:1.0:1775764896.324915:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325039:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000002764acac ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325047:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325052:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.325059:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.325060:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.325062:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.325070:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.325072:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.325073:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2095b8430 (ffff90c2195ee878/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.325075:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325080:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.325086:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2195ea5d8 (ffff90c2195ee758/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.325087:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2095b8430 (ffff90c2195ee758/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.325090:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.325092:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24461, 24461] 00000080:00200000:2.0:1775764896.325100:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.325102:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.325154:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100196352 i_blocks 90112 00000080:00200000:2.0:1775764896.325158:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.325175:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100196352, count: 4096 00000080:00200000:2.0:1775764896.325186:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100196352, bytes: 4096 00000080:00200000:2.0:1775764896.325189:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.325193:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24463, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.325194:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100196352, 100200448) 0 0 00000080:00200000:2.0:1775764896.325197:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.325203:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24462, 24462] 00020000:00200000:2.0:1775764896.325205:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100196352 chunk: [100196352, 100200448) 100200448, 4096 00020000:00200000:2.0:1775764896.325208:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.325224:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24462, 24462] 00000008:00010000:2.0:1775764896.325228:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.325230:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.325236:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325242:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100196352 100200447) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325259:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325266:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325272:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.325281:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100196352, 100200448) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.325286:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24462 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.325304:0:18302:0:(rw26.c:858:ll_write_end()) pos 100196352, len 4096, copied 4096 00000080:00200000:2.0:1775764896.325308:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ac0deb6a[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.325311:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e197bddb 17ffffc0001001 3:0 ffff90c212822d20 24462 lru 00000080:00200000:2.0:1775764896.325316:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000965a4626 24462: 1< 0 - - > 2< 100196352 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 417792 - | + - + - > 5< + - + - | 0 - | 3982 - -> 00000080:00200000:2.0:1775764896.325318:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ac0deb6a 00000080:00200000:2.0:1775764896.325319:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.325321:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.325329:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325337:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325349:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325357:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.325368:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.325371:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.325374:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.325384:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.325387:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.325389:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.325393:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325400:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.325409:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.325411:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.325415:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.325418:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24462, 24462] 00000080:00200000:2.0:1775764896.325429:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.325433:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.325463:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100200448 i_blocks 90112 00000080:00200000:2.0:1775764896.325466:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.325481:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100200448, count: 4096 00000080:00200000:2.0:1775764896.325491:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100200448, bytes: 4096 00000080:00200000:2.0:1775764896.325494:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.325497:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24464, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.325499:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100200448, 100204544) 0 0 00000080:00200000:2.0:1775764896.325501:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.325506:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24463, 24463] 00020000:00200000:2.0:1775764896.325509:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100200448 chunk: [100200448, 100204544) 100204544, 4096 00020000:00200000:2.0:1775764896.325510:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.325521:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24463, 24463] 00000008:00010000:2.0:1775764896.325527:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.325530:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.325538:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325546:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100200448 100204543) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325555:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325567:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325573:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.325581:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100200448, 100204544) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.325587:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24463 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.325604:0:18302:0:(rw26.c:858:ll_write_end()) pos 100200448, len 4096, copied 4096 00000080:00200000:2.0:1775764896.325608:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008e84af3a[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.325611:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000efb74e0a 17ffffc0001001 3:0 ffff90c212822380 24463 lru 00000080:00200000:2.0:1775764896.325617:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000210fc7f1 24463: 1< 0 - - > 2< 100200448 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 413696 - | + - + - > 5< + - + - | 0 - | 3983 - -> 00000080:00200000:2.0:1775764896.325619:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008e84af3a 00000080:00200000:2.0:1775764896.325620:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.325622:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.325630:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325638:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325645:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325653:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.325669:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.325683:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.325685:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.325695:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.325697:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.325700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.325704:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325712:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.325720:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.325723:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.325726:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.325728:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24463, 24463] 00000080:00200000:2.0:1775764896.325735:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.325737:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.325757:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100204544 i_blocks 90112 00000080:00200000:2.0:1775764896.325759:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.325770:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100204544, count: 4096 00000080:00200000:2.0:1775764896.325786:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100204544, bytes: 4096 00000080:00200000:2.0:1775764896.325788:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.325791:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24465, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.325792:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100204544, 100208640) 0 0 00000080:00200000:2.0:1775764896.325793:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.325797:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24464, 24464] 00020000:00200000:2.0:1775764896.325799:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100204544 chunk: [100204544, 100208640) 100208640, 4096 00020000:00200000:2.0:1775764896.325801:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.325808:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24464, 24464] 00000008:00010000:2.0:1775764896.325811:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.325813:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.325819:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325827:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100204544 100208639) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325836:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325843:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325849:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.325858:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100204544, 100208640) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.325863:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24464 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.325897:0:18302:0:(rw26.c:858:ll_write_end()) pos 100204544, len 4096, copied 4096 00000080:00200000:2.0:1775764896.325900:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a0765170[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.325902:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000028ed5e31 17ffffc0001001 3:0 ffff90c212822620 24464 lru 00000080:00200000:2.0:1775764896.325906:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000683d08a 24464: 1< 0 - - > 2< 100204544 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 409600 - | + - + - > 5< + - + - | 0 - | 3984 - -> 00000080:00200000:2.0:1775764896.325907:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a0765170 00000080:00200000:2.0:1775764896.325908:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.325909:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.325915:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325920:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325924:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.325929:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.325938:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.325941:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.325943:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.325952:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.325954:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.325956:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.325960:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.325968:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.325976:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.325978:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.325982:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.325985:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24464, 24464] 00000080:00200000:2.0:1775764896.325996:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.325999:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.326019:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100208640 i_blocks 90112 00000080:00200000:2.0:1775764896.326022:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.326033:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100208640, count: 4096 00000080:00200000:2.0:1775764896.326039:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100208640, bytes: 4096 00000080:00200000:2.0:1775764896.326041:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.326043:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24466, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.326044:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100208640, 100212736) 0 0 00000080:00200000:2.0:1775764896.326046:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.326049:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24465, 24465] 00020000:00200000:2.0:1775764896.326051:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100208640 chunk: [100208640, 100212736) 100212736, 4096 00020000:00200000:2.0:1775764896.326053:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.326060:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24465, 24465] 00000008:00010000:2.0:1775764896.326063:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.326065:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.326070:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326076:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100208640 100212735) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326082:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326086:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326090:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.326095:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100208640, 100212736) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.326099:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24465 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.326111:0:18302:0:(rw26.c:858:ll_write_end()) pos 100208640, len 4096, copied 4096 00000080:00200000:2.0:1775764896.326124:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001546308b[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.326126:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c4fd1fc4 17ffffc0001001 3:0 ffff90c212822e00 24465 lru 00000080:00200000:2.0:1775764896.326130:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000041fc28c4 24465: 1< 0 - - > 2< 100208640 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 405504 - | + - + - > 5< + - + - | 0 - | 3985 - -> 00000080:00200000:2.0:1775764896.326131:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001546308b 00000080:00200000:2.0:1775764896.326132:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.326134:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.326139:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326144:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326148:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326153:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.326160:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.326161:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.326163:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.326169:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.326170:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.326172:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.326175:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326180:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.326185:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.326187:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.326189:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.326191:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24465, 24465] 00000080:00200000:2.0:1775764896.326198:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.326200:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.326218:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100212736 i_blocks 90112 00000080:00200000:2.0:1775764896.326221:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.326231:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100212736, count: 4096 00000080:00200000:2.0:1775764896.326237:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100212736, bytes: 4096 00000080:00200000:2.0:1775764896.326239:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.326241:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24467, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.326242:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100212736, 100216832) 0 0 00000080:00200000:2.0:1775764896.326244:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.326249:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24466, 24466] 00020000:00200000:2.0:1775764896.326252:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100212736 chunk: [100212736, 100216832) 100216832, 4096 00020000:00200000:2.0:1775764896.326254:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.326265:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24466, 24466] 00000008:00010000:2.0:1775764896.326270:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.326272:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.326280:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326289:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100212736 100216831) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326298:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326303:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326307:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.326312:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100212736, 100216832) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.326316:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24466 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.326329:0:18302:0:(rw26.c:858:ll_write_end()) pos 100212736, len 4096, copied 4096 00000080:00200000:2.0:1775764896.326331:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001da4d031[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.326333:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000085bd32b2 17ffffc0001001 3:0 ffff90c212822540 24466 lru 00000080:00200000:2.0:1775764896.326336:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000011ab76e0 24466: 1< 0 - - > 2< 100212736 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 401408 - | + - + - > 5< + - + - | 0 - | 3986 - -> 00000080:00200000:2.0:1775764896.326337:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001da4d031 00000080:00200000:2.0:1775764896.326338:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.326340:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.326344:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326350:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326354:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326362:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.326368:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.326370:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.326371:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.326377:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.326379:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.326381:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.326383:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326388:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.326393:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.326394:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.326397:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.326398:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24466, 24466] 00000080:00200000:2.0:1775764896.326405:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.326407:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.326425:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100216832 i_blocks 90112 00000080:00200000:2.0:1775764896.326428:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.326438:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100216832, count: 4096 00000080:00200000:2.0:1775764896.326444:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100216832, bytes: 4096 00000080:00200000:2.0:1775764896.326445:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.326448:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24468, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.326449:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100216832, 100220928) 0 0 00000080:00200000:2.0:1775764896.326451:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.326454:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24467, 24467] 00020000:00200000:2.0:1775764896.326456:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100216832 chunk: [100216832, 100220928) 100220928, 4096 00020000:00200000:2.0:1775764896.326458:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.326464:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24467, 24467] 00000008:00010000:2.0:1775764896.326467:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.326469:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.326474:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326481:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100216832 100220927) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326490:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326497:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326504:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.326513:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100216832, 100220928) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.326518:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24467 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.326534:0:18302:0:(rw26.c:858:ll_write_end()) pos 100216832, len 4096, copied 4096 00000080:00200000:2.0:1775764896.326538:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b81782f2[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.326541:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ef10e658 17ffffc0001001 3:0 ffff90c212822460 24467 lru 00000080:00200000:2.0:1775764896.326547:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000368bd2b 24467: 1< 0 - - > 2< 100216832 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 397312 - | + - + - > 5< + - + - | 0 - | 3987 - -> 00000080:00200000:2.0:1775764896.326549:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b81782f2 00000080:00200000:2.0:1775764896.326550:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.326552:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.326560:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326568:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326574:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326582:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.326593:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.326596:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.326598:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.326607:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.326610:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.326612:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.326616:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326623:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.326636:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.326638:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.326642:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.326645:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24467, 24467] 00000080:00200000:2.0:1775764896.326654:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.326656:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.326676:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100220928 i_blocks 90112 00000080:00200000:2.0:1775764896.326678:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.326689:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100220928, count: 4096 00000080:00200000:2.0:1775764896.326695:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100220928, bytes: 4096 00000080:00200000:2.0:1775764896.326697:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.326699:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24469, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.326700:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100220928, 100225024) 0 0 00000080:00200000:2.0:1775764896.326702:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.326705:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24468, 24468] 00020000:00200000:2.0:1775764896.326707:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100220928 chunk: [100220928, 100225024) 100225024, 4096 00020000:00200000:2.0:1775764896.326708:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.326715:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24468, 24468] 00000008:00010000:2.0:1775764896.326719:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.326720:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.326726:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326732:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100220928 100225023) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326737:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326746:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326750:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.326754:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100220928, 100225024) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.326757:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24468 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.326769:0:18302:0:(rw26.c:858:ll_write_end()) pos 100220928, len 4096, copied 4096 00000080:00200000:2.0:1775764896.326771:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000099bc8040[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.326773:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c7264aa0 17ffffc0001001 3:0 ffff90c212822a80 24468 lru 00000080:00200000:2.0:1775764896.326776:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b1ca3233 24468: 1< 0 - - > 2< 100220928 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 393216 - | + - + - > 5< + - + - | 0 - | 3988 - -> 00000080:00200000:2.0:1775764896.326777:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000099bc8040 00000080:00200000:2.0:1775764896.326778:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.326779:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.326784:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326789:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326793:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.326797:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.326808:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.326809:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.326811:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.326817:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.326819:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.326820:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.326823:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326828:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.326833:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.326834:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.326836:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.326838:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24468, 24468] 00000080:00200000:2.0:1775764896.326845:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.326847:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.326865:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100225024 i_blocks 90112 00000080:00200000:2.0:1775764896.326867:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.326895:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100225024, count: 4096 00000080:00200000:2.0:1775764896.326902:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100225024, bytes: 4096 00000080:00200000:2.0:1775764896.326904:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.326906:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24470, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.326907:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100225024, 100229120) 0 0 00000080:00200000:2.0:1775764896.326909:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.326912:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24469, 24469] 00020000:00200000:2.0:1775764896.326915:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100225024 chunk: [100225024, 100229120) 100229120, 4096 00020000:00200000:2.0:1775764896.326916:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.326923:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24469, 24469] 00000008:00010000:2.0:1775764896.326926:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.326927:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.326934:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326942:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100225024 100229119) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326950:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326957:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.326964:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.326972:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100225024, 100229120) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.326977:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24469 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.326994:0:18302:0:(rw26.c:858:ll_write_end()) pos 100225024, len 4096, copied 4096 00000080:00200000:2.0:1775764896.326999:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000363b43d1[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.327001:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003460b9d3 17ffffc0001001 3:0 ffff90c212822700 24469 lru 00000080:00200000:2.0:1775764896.327007:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000089f8aba2 24469: 1< 0 - - > 2< 100225024 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 389120 - | + - + - > 5< + - + - | 0 - | 3989 - -> 00000080:00200000:2.0:1775764896.327009:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000363b43d1 00000080:00200000:2.0:1775764896.327010:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.327011:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.327017:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327022:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327027:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327031:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.327038:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.327039:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.327041:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.327048:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.327049:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.327051:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.327054:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327059:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.327064:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.327065:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.327067:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.327069:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24469, 24469] 00000080:00200000:2.0:1775764896.327076:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.327078:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.327097:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100229120 i_blocks 90112 00000080:00200000:2.0:1775764896.327099:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.327110:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100229120, count: 4096 00000080:00200000:2.0:1775764896.327129:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100229120, bytes: 4096 00000080:00200000:2.0:1775764896.327131:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.327134:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24471, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.327135:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100229120, 100233216) 0 0 00000080:00200000:2.0:1775764896.327137:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.327140:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24470, 24470] 00020000:00200000:2.0:1775764896.327142:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100229120 chunk: [100229120, 100233216) 100233216, 4096 00020000:00200000:2.0:1775764896.327143:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.327150:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24470, 24470] 00000008:00010000:2.0:1775764896.327154:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.327155:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.327161:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327167:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100229120 100233215) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327172:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327177:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327181:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.327186:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100229120, 100233216) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.327193:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24470 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.327206:0:18302:0:(rw26.c:858:ll_write_end()) pos 100229120, len 4096, copied 4096 00000080:00200000:2.0:1775764896.327208:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e78ba908[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.327210:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000633c9e77 17ffffc0001001 3:0 ffff90c212822c40 24470 lru 00000080:00200000:2.0:1775764896.327214:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008922270a 24470: 1< 0 - - > 2< 100229120 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 385024 - | + - + - > 5< + - + - | 0 - | 3990 - -> 00000080:00200000:2.0:1775764896.327215:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e78ba908 00000080:00200000:2.0:1775764896.327216:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.327218:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.327222:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327227:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327231:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327235:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.327242:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.327244:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.327245:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.327254:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.327256:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.327258:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.327262:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327269:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.327277:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.327279:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.327283:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.327285:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24470, 24470] 00000080:00200000:2.0:1775764896.327296:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.327299:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.327320:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100233216 i_blocks 90112 00000080:00200000:2.0:1775764896.327323:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.327333:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100233216, count: 4096 00000080:00200000:2.0:1775764896.327339:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100233216, bytes: 4096 00000080:00200000:2.0:1775764896.327341:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.327343:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24472, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.327345:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100233216, 100237312) 0 0 00000080:00200000:2.0:1775764896.327346:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.327350:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24471, 24471] 00020000:00200000:2.0:1775764896.327352:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100233216 chunk: [100233216, 100237312) 100237312, 4096 00020000:00200000:2.0:1775764896.327354:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.327360:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24471, 24471] 00000008:00010000:2.0:1775764896.327363:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.327365:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.327370:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327376:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100233216 100237311) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327382:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327386:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327390:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.327395:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100233216, 100237312) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.327398:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24471 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.327410:0:18302:0:(rw26.c:858:ll_write_end()) pos 100233216, len 4096, copied 4096 00000080:00200000:2.0:1775764896.327413:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001024a6f5[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.327415:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000064dcf068 17ffffc0001001 3:0 ffff90c2128228c0 24471 lru 00000080:00200000:2.0:1775764896.327419:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a4d03880 24471: 1< 0 - - > 2< 100233216 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 380928 - | + - + - > 5< + - + - | 0 - | 3991 - -> 00000080:00200000:2.0:1775764896.327420:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001024a6f5 00000080:00200000:2.0:1775764896.327421:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.327422:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.327427:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327432:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327436:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327444:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.327451:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.327452:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.327454:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.327460:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.327461:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.327463:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.327465:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327470:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.327476:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.327477:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.327479:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.327481:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24471, 24471] 00000080:00200000:2.0:1775764896.327488:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.327489:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.327508:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100237312 i_blocks 90112 00000080:00200000:2.0:1775764896.327510:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.327520:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100237312, count: 4096 00000080:00200000:2.0:1775764896.327526:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100237312, bytes: 4096 00000080:00200000:2.0:1775764896.327528:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.327530:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24473, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.327531:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100237312, 100241408) 0 0 00000080:00200000:2.0:1775764896.327533:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.327537:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24472, 24472] 00020000:00200000:2.0:1775764896.327539:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100237312 chunk: [100237312, 100241408) 100241408, 4096 00020000:00200000:2.0:1775764896.327541:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.327551:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24472, 24472] 00000008:00010000:2.0:1775764896.327556:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.327559:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.327566:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327575:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100237312 100241407) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327583:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327591:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327598:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.327607:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100237312, 100241408) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.327611:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24472 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.327629:0:18302:0:(rw26.c:858:ll_write_end()) pos 100237312, len 4096, copied 4096 00000080:00200000:2.0:1775764896.327632:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006c5ad69d[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.327634:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008b27bafc 17ffffc0001021 2:0 ffff90c212822b60 24472 lru 00000080:00200000:2.0:1775764896.327638:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000657f74f9 24472: 1< 0 - - > 2< 100237312 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 376832 - | + - + - > 5< + - + - | 0 - | 3992 - -> 00000080:00200000:2.0:1775764896.327639:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006c5ad69d 00000080:00200000:2.0:1775764896.327640:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.327641:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.327646:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327651:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327655:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327659:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.327666:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.327667:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.327669:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.327675:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.327676:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.327678:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.327680:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327686:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.327694:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.327696:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.327698:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.327700:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24472, 24472] 00000080:00200000:2.0:1775764896.327707:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.327709:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.327728:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100241408 i_blocks 90112 00000080:00200000:2.0:1775764896.327730:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.327741:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100241408, count: 4096 00000080:00200000:2.0:1775764896.327747:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100241408, bytes: 4096 00000080:00200000:2.0:1775764896.327748:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.327751:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24474, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.327752:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100241408, 100245504) 0 0 00000080:00200000:2.0:1775764896.327754:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.327757:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24473, 24473] 00020000:00200000:2.0:1775764896.327759:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100241408 chunk: [100241408, 100245504) 100245504, 4096 00020000:00200000:2.0:1775764896.327760:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.327766:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24473, 24473] 00000008:00010000:2.0:1775764896.327770:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.327771:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.327776:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327782:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100241408 100245503) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327788:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327796:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.327800:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.327804:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100241408, 100245504) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.327807:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24473 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.327819:0:18302:0:(rw26.c:858:ll_write_end()) pos 100241408, len 4096, copied 4096 00000080:00200000:2.0:1775764896.327822:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005bac8899[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.327823:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000061fcab7e 17ffffc0001001 3:0 ffff90c212822000 24473 lru 00000080:00200000:2.0:1775764896.327826:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000434d4fd7 24473: 1< 0 - - > 2< 100241408 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 372736 - | + - + - > 5< + - + - | 0 - | 3993 - -> 00000080:00200000:2.0:1775764896.327827:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005bac8899 00000080:00200000:2.0:1775764896.327828:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.327829:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.327834:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327839:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327843:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.327847:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.327857:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.327858:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.327860:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.327866:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.327868:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.329441:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.329452:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.329467:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.329482:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.329485:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.329495:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.329503:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24473, 24473] 00000080:00200000:2.0:1775764896.329529:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.329534:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.329612:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100245504 i_blocks 90112 00000080:00200000:2.0:1775764896.329618:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.329646:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100245504, count: 4096 00000080:00200000:2.0:1775764896.329660:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100245504, bytes: 4096 00000080:00200000:2.0:1775764896.329665:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.329670:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24475, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.329672:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100245504, 100249600) 0 0 00000080:00200000:2.0:1775764896.329676:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.329687:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24474, 24474] 00020000:00200000:2.0:1775764896.329693:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100245504 chunk: [100245504, 100249600) 100249600, 4096 00020000:00200000:2.0:1775764896.329697:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.329713:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24474, 24474] 00000008:00010000:2.0:1775764896.329723:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.329726:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.329744:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.329756:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100245504 100249599) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.329766:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.329774:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.329781:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.329792:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100245504, 100249600) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.329802:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24474 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.329843:0:18302:0:(rw26.c:858:ll_write_end()) pos 100245504, len 4096, copied 4096 00000080:00200000:2.0:1775764896.329850:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c7f76edb[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.329853:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000140fcd9f 17ffffc0001001 3:0 ffff90c212822ee0 24474 lru 00000080:00200000:2.0:1775764896.329861:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000018dbe3ff 24474: 1< 0 - - > 2< 100245504 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 368640 - | + - + - > 5< + - + - | 0 - | 3994 - -> 00000080:00200000:2.0:1775764896.329863:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c7f76edb 00000080:00200000:2.0:1775764896.329864:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.329868:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.329900:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.329909:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.329917:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.329925:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.329954:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.329959:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.329962:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.329980:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.329983:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.329986:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.329991:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.329999:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.330009:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.330011:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.330015:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.330018:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24474, 24474] 00000080:00200000:2.0:1775764896.330029:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.330032:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.330060:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100249600 i_blocks 90112 00000080:00200000:2.0:1775764896.330064:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.330080:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100249600, count: 4096 00000080:00200000:2.0:1775764896.330090:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100249600, bytes: 4096 00000080:00200000:2.0:1775764896.330093:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.330097:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24476, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.330098:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100249600, 100253696) 0 0 00000080:00200000:2.0:1775764896.330101:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.330106:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24475, 24475] 00020000:00200000:2.0:1775764896.330110:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100249600 chunk: [100249600, 100253696) 100253696, 4096 00020000:00200000:2.0:1775764896.330124:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.330138:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24475, 24475] 00000008:00010000:2.0:1775764896.330144:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.330146:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.330155:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.330165:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100249600 100253695) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.330173:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.330182:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.330189:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.330198:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100249600, 100253696) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.330203:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24475 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.330221:0:18302:0:(rw26.c:858:ll_write_end()) pos 100249600, len 4096, copied 4096 00000080:00200000:2.0:1775764896.330226:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f29b712d[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.330229:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000019e5d90f 17ffffc0001001 3:0 ffff90c2128220e0 24475 lru 00000080:00200000:2.0:1775764896.330235:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001b8bab59 24475: 1< 0 - - > 2< 100249600 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 364544 - | + - + - > 5< + - + - | 0 - | 3995 - -> 00000080:00200000:2.0:1775764896.330237:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f29b712d 00000080:00200000:2.0:1775764896.330239:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.330241:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.330260:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.330269:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.330277:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.330285:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.330297:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.330300:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.332601:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.332650:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.332655:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.332659:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.332749:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.332767:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.332915:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.332918:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.332939:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.332946:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24475, 24475] 00000080:00200000:2.0:1775764896.332976:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.332979:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.333044:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100253696 i_blocks 90112 00000080:00200000:2.0:1775764896.333050:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.333083:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100253696, count: 4096 00000080:00200000:2.0:1775764896.333097:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100253696, bytes: 4096 00000080:00200000:2.0:1775764896.333101:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.333107:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24477, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.333109:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100253696, 100257792) 0 0 00000080:00200000:2.0:1775764896.333123:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.333136:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24476, 24476] 00020000:00200000:2.0:1775764896.333143:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100253696 chunk: [100253696, 100257792) 100257792, 4096 00020000:00200000:2.0:1775764896.333146:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.333165:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24476, 24476] 00000008:00010000:2.0:1775764896.333176:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.333180:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.333198:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.333211:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100253696 100257791) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.333222:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.333230:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.333237:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.333247:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100253696, 100257792) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.333260:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24476 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.333304:0:18302:0:(rw26.c:858:ll_write_end()) pos 100253696, len 4096, copied 4096 00000080:00200000:2.0:1775764896.333312:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000098c56ec1[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.333315:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000849ffb8e 17ffffc0001001 3:0 ffff90c2128221c0 24476 lru 00000080:00200000:2.0:1775764896.333321:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000005d5692bd 24476: 1< 0 - - > 2< 100253696 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 360448 - | + - + - > 5< + - + - | 0 - | 3996 - -> 00000080:00200000:2.0:1775764896.333323:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000098c56ec1 00000080:00200000:2.0:1775764896.333324:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.333329:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.333347:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.333356:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.333363:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.333377:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.333397:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.333401:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.333404:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.333416:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.333419:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.333422:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.333426:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.333433:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.333442:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.333445:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.333449:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.333452:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24476, 24476] 00000080:00200000:2.0:1775764896.333463:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.333466:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.333860:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100257792 i_blocks 90112 00000080:00200000:2.0:1775764896.333866:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.333911:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100257792, count: 4096 00000080:00200000:2.0:1775764896.333923:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100257792, bytes: 4096 00000080:00200000:2.0:1775764896.333936:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.333941:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24478, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.333942:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100257792, 100261888) 0 0 00000080:00200000:2.0:1775764896.333945:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.333952:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24477, 24477] 00020000:00200000:2.0:1775764896.333956:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100257792 chunk: [100257792, 100261888) 100261888, 4096 00020000:00200000:2.0:1775764896.333959:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.333972:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24477, 24477] 00000008:00010000:2.0:1775764896.333978:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.333982:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.333991:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.334001:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100257792 100261887) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.334010:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.334017:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.334024:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.334033:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100257792, 100261888) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.334038:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24477 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.334071:0:18302:0:(rw26.c:858:ll_write_end()) pos 100257792, len 4096, copied 4096 00000080:00200000:2.0:1775764896.334076:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000042d413ab[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.334079:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000657ef0c4 17ffffc0001001 3:0 ffff90c2128238c0 24477 lru 00000080:00200000:2.0:1775764896.334086:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ee202200 24477: 1< 0 - - > 2< 100257792 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 356352 - | + - + - > 5< + - + - | 0 - | 3997 - -> 00000080:00200000:2.0:1775764896.334088:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000042d413ab 00000080:00200000:2.0:1775764896.334089:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.334092:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.334102:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.334110:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.334131:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.334138:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.334151:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.334154:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.334157:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.334169:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.334172:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.334175:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.334179:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.334187:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.334201:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.334203:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.334208:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.334212:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24477, 24477] 00000080:00200000:2.0:1775764896.334223:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.334225:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.334257:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100261888 i_blocks 90112 00000080:00200000:2.0:1775764896.334261:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.334277:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100261888, count: 4096 00000080:00200000:2.0:1775764896.334288:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100261888, bytes: 4096 00000080:00200000:2.0:1775764896.334291:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.334294:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24479, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.334296:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100261888, 100265984) 0 0 00000080:00200000:2.0:1775764896.334298:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.334303:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24478, 24478] 00020000:00200000:2.0:1775764896.334306:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100261888 chunk: [100261888, 100265984) 100265984, 4096 00020000:00200000:2.0:1775764896.334308:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.334319:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24478, 24478] 00000008:00010000:2.0:1775764896.334324:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.334327:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.334334:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.334343:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100261888 100265983) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.334351:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.334365:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.334372:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.334380:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100261888, 100265984) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.334386:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24478 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.334406:0:18302:0:(rw26.c:858:ll_write_end()) pos 100261888, len 4096, copied 4096 00000080:00200000:2.0:1775764896.334411:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cff35d07[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.334414:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000eae35672 17ffffc0001001 3:0 ffff90c212823b60 24478 lru 00000080:00200000:2.0:1775764896.334451:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009744842f 24478: 1< 0 - - > 2< 100261888 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 352256 - | + - + - > 5< + - + - | 0 - | 3998 - -> 00000080:00200000:2.0:1775764896.334453:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cff35d07 00000080:00200000:2.0:1775764896.334454:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.334457:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.334466:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.334474:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.335021:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.335030:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.335052:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.335056:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.335059:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.335072:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.335075:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.335078:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.335082:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.335092:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.335101:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.335104:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.335109:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.335125:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24478, 24478] 00000080:00200000:2.0:1775764896.335140:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.335143:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.335183:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100265984 i_blocks 90112 00000080:00200000:2.0:1775764896.335187:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.335207:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100265984, count: 4096 00000080:00200000:2.0:1775764896.335219:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100265984, bytes: 4096 00000080:00200000:2.0:1775764896.335222:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.335227:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24480, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.335228:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100265984, 100270080) 0 0 00000080:00200000:2.0:1775764896.335231:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.335238:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24479, 24479] 00020000:00200000:2.0:1775764896.335242:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100265984 chunk: [100265984, 100270080) 100270080, 4096 00020000:00200000:2.0:1775764896.335245:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.335257:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24479, 24479] 00000008:00010000:2.0:1775764896.335264:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.335268:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.335278:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.335288:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100265984 100270079) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.335295:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.335301:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.335308:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.335316:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100265984, 100270080) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.335322:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24479 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.335345:0:18302:0:(rw26.c:858:ll_write_end()) pos 100265984, len 4096, copied 4096 00000080:00200000:2.0:1775764896.335350:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ca84cb16[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.335353:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d8a34394 17ffffc0001001 3:0 ffff90c212823000 24479 lru 00000080:00200000:2.0:1775764896.335358:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000395f4546 24479: 1< 0 - - > 2< 100265984 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 348160 - | + - + - > 5< + - + - | 0 - | 3999 - -> 00000080:00200000:2.0:1775764896.335361:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ca84cb16 00000080:00200000:2.0:1775764896.335362:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.335364:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.335373:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.335382:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.335389:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.335397:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.335409:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.335412:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.335414:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.335424:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.335427:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.335430:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.335433:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.335441:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.335450:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.335452:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.335457:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.335459:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24479, 24479] 00000080:00200000:2.0:1775764896.335471:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.335474:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.336344:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100270080 i_blocks 90112 00000080:00200000:2.0:1775764896.336349:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.336374:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100270080, count: 4096 00000080:00200000:2.0:1775764896.336387:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100270080, bytes: 4096 00000080:00200000:2.0:1775764896.336390:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.336396:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24481, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.336398:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100270080, 100274176) 0 0 00000080:00200000:2.0:1775764896.336401:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.336410:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24480, 24480] 00020000:00200000:2.0:1775764896.336415:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100270080 chunk: [100270080, 100274176) 100274176, 4096 00020000:00200000:2.0:1775764896.336418:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.336434:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24480, 24480] 00000008:00010000:2.0:1775764896.336442:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.336446:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.336460:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.336472:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100270080 100274175) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.336496:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.336504:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.336511:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.336521:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100270080, 100274176) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.336529:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24480 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.336567:0:18302:0:(rw26.c:858:ll_write_end()) pos 100270080, len 4096, copied 4096 00000080:00200000:2.0:1775764896.336573:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000095eb9812[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.336576:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002667ceee 17ffffc0001001 3:0 ffff90c212823ee0 24480 lru 00000080:00200000:2.0:1775764896.336582:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001c6a1ac3 24480: 1< 0 - - > 2< 100270080 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 344064 - | + - + - > 5< + - + - | 0 - | 4000 - -> 00000080:00200000:2.0:1775764896.336584:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000095eb9812 00000080:00200000:2.0:1775764896.336585:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.336588:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.336600:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.336608:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.336615:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.336623:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.336636:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.336639:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.336642:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.336655:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.336658:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.336661:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.336665:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.336686:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.336697:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.336700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.336705:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.336709:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24480, 24480] 00000080:00200000:2.0:1775764896.336720:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.336723:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.336754:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100274176 i_blocks 90112 00000080:00200000:2.0:1775764896.336764:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.336791:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100274176, count: 4096 00000080:00200000:2.0:1775764896.336803:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100274176, bytes: 4096 00000080:00200000:2.0:1775764896.336805:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.336809:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24482, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.336811:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100274176, 100278272) 0 0 00000080:00200000:2.0:1775764896.336813:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.336818:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24481, 24481] 00020000:00200000:2.0:1775764896.336821:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100274176 chunk: [100274176, 100278272) 100278272, 4096 00020000:00200000:2.0:1775764896.336823:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.336834:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24481, 24481] 00000008:00010000:2.0:1775764896.336840:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.336843:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.336851:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.336861:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100274176 100278271) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.336868:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.336896:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.336902:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.336910:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100274176, 100278272) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.336916:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24481 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.336946:0:18302:0:(rw26.c:858:ll_write_end()) pos 100274176, len 4096, copied 4096 00000080:00200000:2.0:1775764896.336951:0:18302:0:(rw26.c:916:ll_write_end()) page@000000005e933783[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.336954:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000051537ede 17ffffc0001001 3:0 ffff90c2128230e0 24481 lru 00000080:00200000:2.0:1775764896.336959:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000141106b4 24481: 1< 0 - - > 2< 100274176 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 339968 - | + - + - > 5< + - + - | 0 - | 4001 - -> 00000080:00200000:2.0:1775764896.336961:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000005e933783 00000080:00200000:2.0:1775764896.336962:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.336964:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.336973:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.336981:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.336989:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337002:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.337015:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.337018:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.337020:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.337031:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.337034:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.337036:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.337041:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337049:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.337057:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.337059:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.337064:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.337067:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24481, 24481] 00000080:00200000:2.0:1775764896.337078:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.337081:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.337125:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100278272 i_blocks 90112 00000080:00200000:2.0:1775764896.337129:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.337146:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100278272, count: 4096 00000080:00200000:2.0:1775764896.337157:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100278272, bytes: 4096 00000080:00200000:2.0:1775764896.337160:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.337164:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24483, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.337166:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100278272, 100282368) 0 0 00000080:00200000:2.0:1775764896.337169:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.337175:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24482, 24482] 00020000:00200000:2.0:1775764896.337179:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100278272 chunk: [100278272, 100282368) 100282368, 4096 00020000:00200000:2.0:1775764896.337181:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.337194:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24482, 24482] 00000008:00010000:2.0:1775764896.337200:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.337203:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.337211:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337221:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100278272 100282367) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337231:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337239:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337247:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.337255:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100278272, 100282368) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.337261:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24482 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.337278:0:18302:0:(rw26.c:858:ll_write_end()) pos 100278272, len 4096, copied 4096 00000080:00200000:2.0:1775764896.337282:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000e0acecf[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.337285:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000376ccb00 17ffffc0001001 3:0 ffff90c2128231c0 24482 lru 00000080:00200000:2.0:1775764896.337291:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000049fc7fb8 24482: 1< 0 - - > 2< 100278272 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 335872 - | + - + - > 5< + - + - | 0 - | 4002 - -> 00000080:00200000:2.0:1775764896.337293:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000e0acecf 00000080:00200000:2.0:1775764896.337294:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.337296:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.337304:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337312:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337319:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337326:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.337337:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.337340:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.337342:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.337353:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.337356:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.337358:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.337362:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337370:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.337383:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.337385:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.337389:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.337392:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24482, 24482] 00000080:00200000:2.0:1775764896.337403:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.337407:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.337438:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100282368 i_blocks 90112 00000080:00200000:2.0:1775764896.337442:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.337458:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100282368, count: 4096 00000080:00200000:2.0:1775764896.337469:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100282368, bytes: 4096 00000080:00200000:2.0:1775764896.337472:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.337476:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24484, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.337477:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100282368, 100286464) 0 0 00000080:00200000:2.0:1775764896.337480:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.337485:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24483, 24483] 00020000:00200000:2.0:1775764896.337489:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100282368 chunk: [100282368, 100286464) 100286464, 4096 00020000:00200000:2.0:1775764896.337491:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.337504:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24483, 24483] 00000008:00010000:2.0:1775764896.337509:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.337512:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.337520:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337529:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100282368 100286463) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337538:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337551:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337558:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.337566:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100282368, 100286464) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.337571:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24483 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.337589:0:18302:0:(rw26.c:858:ll_write_end()) pos 100282368, len 4096, copied 4096 00000080:00200000:2.0:1775764896.337594:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000819d26f0[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.337598:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007a9ae95d 17ffffc0001001 3:0 ffff90c2128237e0 24483 lru 00000080:00200000:2.0:1775764896.337603:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000869bb848 24483: 1< 0 - - > 2< 100282368 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 331776 - | + - + - > 5< + - + - | 0 - | 4003 - -> 00000080:00200000:2.0:1775764896.337605:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000819d26f0 00000080:00200000:2.0:1775764896.337606:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.337608:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.337616:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337623:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337630:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337637:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.337653:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.337656:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.337659:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.337669:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.337672:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.337675:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.337678:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337686:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.337694:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.337696:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.337701:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.337704:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24483, 24483] 00000080:00200000:2.0:1775764896.337715:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.337718:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.337748:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100286464 i_blocks 90112 00000080:00200000:2.0:1775764896.337752:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.337768:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100286464, count: 4096 00000080:00200000:2.0:1775764896.337778:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100286464, bytes: 4096 00000080:00200000:2.0:1775764896.337781:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.337785:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24485, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.337786:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100286464, 100290560) 0 0 00000080:00200000:2.0:1775764896.337789:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.337793:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24484, 24484] 00020000:00200000:2.0:1775764896.337797:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100286464 chunk: [100286464, 100290560) 100290560, 4096 00020000:00200000:2.0:1775764896.337799:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.337811:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24484, 24484] 00000008:00010000:2.0:1775764896.337817:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.337819:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.337827:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337836:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100286464 100290559) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337844:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337851:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.337857:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.337865:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100286464, 100290560) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.337894:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24484 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.337912:0:18302:0:(rw26.c:858:ll_write_end()) pos 100286464, len 4096, copied 4096 00000080:00200000:2.0:1775764896.337916:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000008a2c2d8[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.337919:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000119c1e10 17ffffc0001001 3:0 ffff90c2128239a0 24484 lru 00000080:00200000:2.0:1775764896.337925:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000087642c70 24484: 1< 0 - - > 2< 100286464 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 327680 - | + - + - > 5< + - + - | 0 - | 4004 - -> 00000080:00200000:2.0:1775764896.337927:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000008a2c2d8 00000080:00200000:2.0:1775764896.337928:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.337930:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.337938:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337946:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337953:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.337960:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.337970:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.337973:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.337975:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.337984:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.337987:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.337990:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.337993:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338001:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.338009:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.338011:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.338015:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.338018:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24484, 24484] 00000080:00200000:2.0:1775764896.338030:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.338033:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.338059:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100290560 i_blocks 90112 00000080:00200000:2.0:1775764896.338063:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.338078:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100290560, count: 4096 00000080:00200000:2.0:1775764896.338088:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100290560, bytes: 4096 00000080:00200000:2.0:1775764896.338090:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.338094:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24486, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.338095:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100290560, 100294656) 0 0 00000080:00200000:2.0:1775764896.338097:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.338102:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24485, 24485] 00020000:00200000:2.0:1775764896.338105:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100290560 chunk: [100290560, 100294656) 100294656, 4096 00020000:00200000:2.0:1775764896.338107:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.338131:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24485, 24485] 00000008:00010000:2.0:1775764896.338137:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.338139:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.338146:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338154:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100290560 100294655) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338161:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338168:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338174:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.338182:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100290560, 100294656) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.338187:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24485 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.338202:0:18302:0:(rw26.c:858:ll_write_end()) pos 100290560, len 4096, copied 4096 00000080:00200000:2.0:1775764896.338206:0:18302:0:(rw26.c:916:ll_write_end()) page@000000006e57655b[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.338209:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000007a169b18 17ffffc0001001 3:0 ffff90c2128232a0 24485 lru 00000080:00200000:2.0:1775764896.338214:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a5668bfc 24485: 1< 0 - - > 2< 100290560 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 323584 - | + - + - > 5< + - + - | 0 - | 4005 - -> 00000080:00200000:2.0:1775764896.338216:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000006e57655b 00000080:00200000:2.0:1775764896.338217:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.338219:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.338226:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.338233:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.338241:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.338248:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.338260:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.338263:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.338265:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.338275:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.338278:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.338280:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.338285:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338292:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.338300:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.338303:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.338307:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.338311:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24485, 24485] 00000080:00200000:2.0:1775764896.338321:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.338325:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.338353:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100294656 i_blocks 90112 00000080:00200000:2.0:1775764896.338356:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.338372:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100294656, count: 4096 00000080:00200000:2.0:1775764896.338383:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100294656, bytes: 4096 00000080:00200000:2.0:1775764896.338386:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.338390:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24487, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.338392:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100294656, 100298752) 0 0 00000080:00200000:2.0:1775764896.338394:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.338399:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24486, 24486] 00020000:00200000:2.0:1775764896.338402:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100294656 chunk: [100294656, 100298752) 100298752, 4096 00020000:00200000:2.0:1775764896.338405:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.338416:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24486, 24486] 00000008:00010000:2.0:1775764896.338422:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.338425:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.338432:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338441:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100294656 100298751) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338449:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338456:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338463:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.338470:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100294656, 100298752) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.338475:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24486 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.338504:0:18302:0:(rw26.c:858:ll_write_end()) pos 100294656, len 4096, copied 4096 00000080:00200000:2.0:1775764896.338508:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000065603101[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.338511:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001cfc646e 17ffffc0001001 3:0 ffff90c212823d20 24486 lru 00000080:00200000:2.0:1775764896.338515:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c2e998a8 24486: 1< 0 - - > 2< 100294656 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 319488 - | + - + - > 5< + - + - | 0 - | 4006 - -> 00000080:00200000:2.0:1775764896.338517:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000065603101 00000080:00200000:2.0:1775764896.338519:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.338521:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.338528:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.338536:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.338543:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.338555:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.338566:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.338569:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.338571:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.338580:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.338583:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.338585:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.338589:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338596:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.338604:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.338607:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.338611:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.338614:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24486, 24486] 00000080:00200000:2.0:1775764896.338625:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.338628:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.338655:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100298752 i_blocks 90112 00000080:00200000:2.0:1775764896.338659:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.338674:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100298752, count: 4096 00000080:00200000:2.0:1775764896.338685:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100298752, bytes: 4096 00000080:00200000:2.0:1775764896.338688:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.338691:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24488, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.338693:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100298752, 100302848) 0 0 00000080:00200000:2.0:1775764896.338695:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.338700:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24487, 24487] 00020000:00200000:2.0:1775764896.338703:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100298752 chunk: [100298752, 100302848) 100302848, 4096 00020000:00200000:2.0:1775764896.338705:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.338715:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24487, 24487] 00000008:00010000:2.0:1775764896.338721:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.338723:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.338730:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338739:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100298752 100302847) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338747:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338754:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338760:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.338768:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100298752, 100302848) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.338773:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24487 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.338798:0:18302:0:(rw26.c:858:ll_write_end()) pos 100298752, len 4096, copied 4096 00000080:00200000:2.0:1775764896.338803:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b8aca185[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.338806:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000082c59230 17ffffc0001021 2:0 ffff90c212823380 24487 lru 00000080:00200000:2.0:1775764896.338812:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007eed9590 24487: 1< 0 - - > 2< 100298752 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 315392 - | + - + - > 5< + - + - | 0 - | 4007 - -> 00000080:00200000:2.0:1775764896.338814:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b8aca185 00000080:00200000:2.0:1775764896.338815:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.338817:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.338825:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.338833:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.338840:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.338847:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.338858:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.338860:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.338863:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.338907:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.338910:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.338912:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.338917:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.338924:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.338936:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.338938:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.338942:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.338945:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24487, 24487] 00000080:00200000:2.0:1775764896.338955:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.338958:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.338988:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100302848 i_blocks 90112 00000080:00200000:2.0:1775764896.338992:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.339008:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100302848, count: 4096 00000080:00200000:2.0:1775764896.339019:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100302848, bytes: 4096 00000080:00200000:2.0:1775764896.339022:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.339026:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24489, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.339027:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100302848, 100306944) 0 0 00000080:00200000:2.0:1775764896.339029:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.339034:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24488, 24488] 00020000:00200000:2.0:1775764896.339037:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100302848 chunk: [100302848, 100306944) 100306944, 4096 00020000:00200000:2.0:1775764896.339039:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.339051:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24488, 24488] 00000008:00010000:2.0:1775764896.339057:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.339059:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.339067:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.339076:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100302848 100306943) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.339084:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.339095:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.339102:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.339109:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100302848, 100306944) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.341395:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24488 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.341461:0:18302:0:(rw26.c:858:ll_write_end()) pos 100302848, len 4096, copied 4096 00000080:00200000:2.0:1775764896.341482:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000066ae757c[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.341486:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f171062e 17ffffc0001001 3:0 ffff90c212823620 24488 lru 00000080:00200000:2.0:1775764896.341494:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000082df93f4 24488: 1< 0 - - > 2< 100302848 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 311296 - | + - + - > 5< + - + - | 0 - | 4008 - -> 00000080:00200000:2.0:1775764896.341497:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000066ae757c 00000080:00200000:2.0:1775764896.341498:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.341503:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.341524:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.342348:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.342362:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.342373:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.342407:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.342414:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.342418:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.342447:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.342450:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.342454:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.342461:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.342472:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.342496:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.342500:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.342508:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.342515:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24488, 24488] 00000080:00200000:2.0:1775764896.343685:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.343694:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1775764896.343773:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100306944 i_blocks 90112 00000080:00200000:2.0:1775764896.343780:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1775764896.343819:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100306944, count: 4096 00000080:00200000:2.0:1775764896.343835:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100306944, bytes: 4096 00000080:00200000:2.0:1775764896.343841:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:2.0:1775764896.343848:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24490, [0x200000403:0x10:0x0] hash] 00000020:00200000:2.0:1775764896.343850:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100306944, 100311040) 0 0 00000080:00200000:2.0:1775764896.343854:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1775764896.343866:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24489, 24489] 00020000:00200000:2.0:1775764896.343890:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100306944 chunk: [100306944, 100311040) 100311040, 4096 00020000:00200000:2.0:1775764896.343895:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1775764896.343936:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24489, 24489] 00000008:00010000:2.0:1775764896.343948:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:2.0:1775764896.343953:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20a501a30, osc lock ffff90c20a90b560, flags 0x0 00010000:00010000:2.0:1775764896.343977:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.343990:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100306944 100311039) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.344001:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.344009:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.344015:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.344026:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100306944, 100311040) append 0 append lockpos 0 00000080:00200000:2.0:1775764896.344036:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24489 of 0 to 4096 bytes 00000080:00200000:2.0:1775764896.344077:0:18302:0:(rw26.c:858:ll_write_end()) pos 100306944, len 4096, copied 4096 00000080:00200000:2.0:1775764896.344083:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000017f06851[3 00000000b99c1f5d 2 1 00000000ec7b5a13] 00000080:00200000:2.0:1775764896.344087:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f74c7bd6 17ffffc0001001 3:0 ffff90c212823e00 24489 lru 00000080:00200000:2.0:1775764896.344093:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000009fe9569 24489: 1< 0 - - > 2< 100306944 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 307200 - | + - + - > 5< + - + - | 0 - | 4009 - -> 00000080:00200000:2.0:1775764896.344095:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000017f06851 00000080:00200000:2.0:1775764896.344097:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:2.0:1775764896.344100:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:2.0:1775764896.346658:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.346679:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000000ce168aa ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.346687:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:2.0:1775764896.346697:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:2.0:1775764896.346731:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:2.0:1775764896.346740:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:2.0:1775764896.346745:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:2.0:1775764896.346773:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_release():245 00000020:00010000:2.0:1775764896.346777:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_cancel():142 00000020:00010000:2.0:1775764896.346781:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a501a30 (ffff90c20361cf38/1) at cl_lock_cancel():142 00010000:00010000:2.0:1775764896.346787:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:2.0:1775764896.346798:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:2.0:1775764896.346812:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2420f77b8 (ffff90c20361c638/0) at cl_lock_fini():72 00000020:00010000:2.0:1775764896.346815:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a501a30 (ffff90c20361c638/1) at cl_lock_fini():72 00000080:00200000:2.0:1775764896.346823:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:2.0:1775764896.346830:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24489, 24489] 00000080:00200000:2.0:1775764896.346859:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1775764896.346864:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.347039:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100311040 i_blocks 90112 00000080:00200000:3.0:1775764896.347046:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.347089:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100311040, count: 4096 00000080:00200000:3.0:1775764896.347129:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100311040, bytes: 4096 00000080:00200000:3.0:1775764896.347135:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.347141:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24491, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.347143:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100311040, 100315136) 0 0 00000080:00200000:3.0:1775764896.347147:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.347159:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24490, 24490] 00020000:00200000:3.0:1775764896.347166:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100311040 chunk: [100311040, 100315136) 100315136, 4096 00020000:00200000:3.0:1775764896.347170:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.347198:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24490, 24490] 00000008:00010000:3.0:1775764896.347212:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.347217:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.347237:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347266:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100311040 100315135) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347277:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347286:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347293:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.347304:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100311040, 100315136) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.347317:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24490 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.347362:0:18302:0:(rw26.c:858:ll_write_end()) pos 100311040, len 4096, copied 4096 00000080:00200000:3.0:1775764896.347370:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004e842620[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.347374:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fa319221 17ffffc0001001 3:0 ffff90c212825d20 24490 lru 00000080:00200000:3.0:1775764896.347382:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b430cc29 24490: 1< 0 - - > 2< 100311040 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 303104 - | + - + - > 5< + - + - | 0 - | 4010 - -> 00000080:00200000:3.0:1775764896.347385:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004e842620 00000080:00200000:3.0:1775764896.347386:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.347390:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.347404:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.347412:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.347419:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.347426:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.347438:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.347441:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.347444:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.347458:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.347461:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.347464:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.347468:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347476:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.347486:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.347488:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.347493:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.347496:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24490, 24490] 00000080:00200000:3.0:1775764896.347511:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.347515:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.347547:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100315136 i_blocks 90112 00000080:00200000:3.0:1775764896.347552:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.347570:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100315136, count: 4096 00000080:00200000:3.0:1775764896.347581:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100315136, bytes: 4096 00000080:00200000:3.0:1775764896.347584:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.347589:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24492, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.347590:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100315136, 100319232) 0 0 00000080:00200000:3.0:1775764896.347593:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.347599:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24491, 24491] 00020000:00200000:3.0:1775764896.347602:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100315136 chunk: [100315136, 100319232) 100319232, 4096 00020000:00200000:3.0:1775764896.347605:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.347619:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24491, 24491] 00000008:00010000:3.0:1775764896.347625:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.347628:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.347637:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347645:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100315136 100319231) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347654:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347668:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347675:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.347683:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100315136, 100319232) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.347689:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24491 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.347707:0:18302:0:(rw26.c:858:ll_write_end()) pos 100315136, len 4096, copied 4096 00000080:00200000:3.0:1775764896.347711:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c494fdea[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.347714:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c1c4e766 17ffffc0001001 3:0 ffff90c212825380 24491 lru 00000080:00200000:3.0:1775764896.347721:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000f6e5b71 24491: 1< 0 - - > 2< 100315136 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 299008 - | + - + - > 5< + - + - | 0 - | 4011 - -> 00000080:00200000:3.0:1775764896.347723:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c494fdea 00000080:00200000:3.0:1775764896.347724:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.347726:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.347734:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.347741:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.347748:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.347755:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.347771:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.347774:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.347777:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.347786:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.347789:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.347791:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.347795:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347802:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.347810:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.347813:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.347816:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.347819:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24491, 24491] 00000080:00200000:3.0:1775764896.347830:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.347833:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.347864:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100319232 i_blocks 90112 00000080:00200000:3.0:1775764896.347868:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.347906:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100319232, count: 4096 00000080:00200000:3.0:1775764896.347916:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100319232, bytes: 4096 00000080:00200000:3.0:1775764896.347919:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.347923:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24493, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.347924:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100319232, 100323328) 0 0 00000080:00200000:3.0:1775764896.347927:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.347932:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24492, 24492] 00020000:00200000:3.0:1775764896.347936:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100319232 chunk: [100319232, 100323328) 100323328, 4096 00020000:00200000:3.0:1775764896.347938:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.347950:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24492, 24492] 00000008:00010000:3.0:1775764896.347955:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.347958:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.347966:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347975:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100319232 100323327) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347983:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347990:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.347997:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.348005:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100319232, 100323328) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.348010:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24492 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.348029:0:18302:0:(rw26.c:858:ll_write_end()) pos 100319232, len 4096, copied 4096 00000080:00200000:3.0:1775764896.348034:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c8f2f209[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.348037:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f02e1dd4 17ffffc0001001 3:0 ffff90c212825620 24492 lru 00000080:00200000:3.0:1775764896.348043:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ff4528a2 24492: 1< 0 - - > 2< 100319232 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 294912 - | + - + - > 5< + - + - | 0 - | 4012 - -> 00000080:00200000:3.0:1775764896.348045:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c8f2f209 00000080:00200000:3.0:1775764896.348047:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.348049:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.348057:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348064:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348070:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348077:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.348088:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.348091:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.348093:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.348103:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.348106:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.348109:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.348125:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348134:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.348142:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.348145:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.348149:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.348152:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24492, 24492] 00000080:00200000:3.0:1775764896.348165:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.348168:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.348199:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100323328 i_blocks 90112 00000080:00200000:3.0:1775764896.348203:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.348219:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100323328, count: 4096 00000080:00200000:3.0:1775764896.348230:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100323328, bytes: 4096 00000080:00200000:3.0:1775764896.348233:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.348237:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24494, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.348238:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100323328, 100327424) 0 0 00000080:00200000:3.0:1775764896.348240:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.348246:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24493, 24493] 00020000:00200000:3.0:1775764896.348249:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100323328 chunk: [100323328, 100327424) 100327424, 4096 00020000:00200000:3.0:1775764896.348251:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.348263:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24493, 24493] 00000008:00010000:3.0:1775764896.348269:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.348271:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.348280:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348289:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100323328 100327423) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348298:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348306:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348314:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.348322:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100323328, 100327424) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.348328:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24493 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.348345:0:18302:0:(rw26.c:858:ll_write_end()) pos 100323328, len 4096, copied 4096 00000080:00200000:3.0:1775764896.348349:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003ffec283[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.348352:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002ece44bc 17ffffc0001001 3:0 ffff90c212825e00 24493 lru 00000080:00200000:3.0:1775764896.348357:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009c3e6d9b 24493: 1< 0 - - > 2< 100323328 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 290816 - | + - + - > 5< + - + - | 0 - | 4013 - -> 00000080:00200000:3.0:1775764896.348359:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003ffec283 00000080:00200000:3.0:1775764896.348360:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.348363:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.348370:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348378:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348385:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348393:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.348406:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.348409:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.348411:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.348420:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.348423:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.348426:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.348431:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348438:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.348446:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.348448:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.348453:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.348456:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24493, 24493] 00000080:00200000:3.0:1775764896.348466:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.348470:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.348500:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100327424 i_blocks 90112 00000080:00200000:3.0:1775764896.348504:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.348521:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100327424, count: 4096 00000080:00200000:3.0:1775764896.348534:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100327424, bytes: 4096 00000080:00200000:3.0:1775764896.348537:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.348540:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24495, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.348542:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100327424, 100331520) 0 0 00000080:00200000:3.0:1775764896.348544:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.348549:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24494, 24494] 00020000:00200000:3.0:1775764896.348551:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100327424 chunk: [100327424, 100331520) 100331520, 4096 00020000:00200000:3.0:1775764896.348554:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.348566:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24494, 24494] 00000008:00010000:3.0:1775764896.348571:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.348574:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.348582:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348590:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100327424 100331519) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348598:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348605:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348612:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.348620:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100327424, 100331520) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.348625:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24494 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.348642:0:18302:0:(rw26.c:858:ll_write_end()) pos 100327424, len 4096, copied 4096 00000080:00200000:3.0:1775764896.348647:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000095917d0f[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.348649:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000cc12f13d 17ffffc0001001 3:0 ffff90c212825540 24494 lru 00000080:00200000:3.0:1775764896.348655:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f96cad11 24494: 1< 0 - - > 2< 100327424 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 286720 - | + - + - > 5< + - + - | 0 - | 4014 - -> 00000080:00200000:3.0:1775764896.348657:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000095917d0f 00000080:00200000:3.0:1775764896.348658:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.348660:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.348669:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348677:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348683:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348695:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.348706:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.348709:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.348711:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.348721:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.348724:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.348726:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.348729:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348737:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.348745:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.348747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.348751:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.348754:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24494, 24494] 00000080:00200000:3.0:1775764896.348764:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.348768:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.348798:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100331520 i_blocks 90112 00000080:00200000:3.0:1775764896.348801:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.348816:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100331520, count: 4096 00000080:00200000:3.0:1775764896.348826:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100331520, bytes: 4096 00000080:00200000:3.0:1775764896.348829:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.348833:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24496, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.348834:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100331520, 100335616) 0 0 00000080:00200000:3.0:1775764896.348837:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.348843:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24495, 24495] 00020000:00200000:3.0:1775764896.348846:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100331520 chunk: [100331520, 100335616) 100335616, 4096 00020000:00200000:3.0:1775764896.348848:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.348859:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24495, 24495] 00000008:00010000:3.0:1775764896.348865:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.348867:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.348897:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348907:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100331520 100335615) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348916:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348922:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.348929:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.348939:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100331520, 100335616) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.348944:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24495 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.348961:0:18302:0:(rw26.c:858:ll_write_end()) pos 100331520, len 4096, copied 4096 00000080:00200000:3.0:1775764896.348966:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f0423a16[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.348969:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e29c7c0e 17ffffc0001001 3:0 ffff90c212825460 24495 lru 00000080:00200000:3.0:1775764896.348975:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a2a6e985 24495: 1< 0 - - > 2< 100331520 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 282624 - | + - + - > 5< + - + - | 0 - | 4015 - -> 00000080:00200000:3.0:1775764896.348977:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f0423a16 00000080:00200000:3.0:1775764896.348978:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.348980:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.348988:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.348996:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349004:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349012:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.349023:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.349025:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.349028:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.349038:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.349041:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.349043:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.349047:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349055:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.349070:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.349072:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.349076:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.349079:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24495, 24495] 00000080:00200000:3.0:1775764896.349090:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.349093:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.349138:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100335616 i_blocks 90112 00000080:00200000:3.0:1775764896.349142:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.349165:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100335616, count: 4096 00000080:00200000:3.0:1775764896.349176:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100335616, bytes: 4096 00000080:00200000:3.0:1775764896.349179:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.349182:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24497, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.349184:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100335616, 100339712) 0 0 00000080:00200000:3.0:1775764896.349186:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.349191:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24496, 24496] 00020000:00200000:3.0:1775764896.349195:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100335616 chunk: [100335616, 100339712) 100339712, 4096 00020000:00200000:3.0:1775764896.349197:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.349209:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24496, 24496] 00000008:00010000:3.0:1775764896.349215:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.349217:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.349225:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349234:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100335616 100339711) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349242:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349254:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349260:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.349268:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100335616, 100339712) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.349273:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24496 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.349296:0:18302:0:(rw26.c:858:ll_write_end()) pos 100335616, len 4096, copied 4096 00000080:00200000:3.0:1775764896.349301:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000045936a76[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.349304:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ba3f8187 17ffffc0001021 2:0 ffff90c212825a80 24496 lru 00000080:00200000:3.0:1775764896.349309:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000084f77e90 24496: 1< 0 - - > 2< 100335616 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 278528 - | + - + - > 5< + - + - | 0 - | 4016 - -> 00000080:00200000:3.0:1775764896.349311:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000045936a76 00000080:00200000:3.0:1775764896.349312:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.349315:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.349323:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349331:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349338:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349344:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.349358:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.349361:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.349364:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.349374:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.349376:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.349378:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.349382:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349389:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.349397:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.349400:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.349404:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.349406:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24496, 24496] 00000080:00200000:3.0:1775764896.349417:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.349426:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.349455:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100339712 i_blocks 90112 00000080:00200000:3.0:1775764896.349458:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.349474:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100339712, count: 4096 00000080:00200000:3.0:1775764896.349485:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100339712, bytes: 4096 00000080:00200000:3.0:1775764896.349488:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.349492:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24498, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.349494:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100339712, 100343808) 0 0 00000080:00200000:3.0:1775764896.349496:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.349501:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24497, 24497] 00020000:00200000:3.0:1775764896.349505:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100339712 chunk: [100339712, 100343808) 100343808, 4096 00020000:00200000:3.0:1775764896.349507:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.349518:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24497, 24497] 00000008:00010000:3.0:1775764896.349523:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.349525:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.349533:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349549:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100339712 100343807) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349557:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349564:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349572:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.349579:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100339712, 100343808) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.349584:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24497 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.349601:0:18302:0:(rw26.c:858:ll_write_end()) pos 100339712, len 4096, copied 4096 00000080:00200000:3.0:1775764896.349605:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000a9ed1a0[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.349607:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000a4dc44b 17ffffc0001001 3:0 ffff90c212825700 24497 lru 00000080:00200000:3.0:1775764896.349612:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000020356a21 24497: 1< 0 - - > 2< 100339712 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 274432 - | + - + - > 5< + - + - | 0 - | 4017 - -> 00000080:00200000:3.0:1775764896.349614:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000a9ed1a0 00000080:00200000:3.0:1775764896.349615:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.349617:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.349625:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349633:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349639:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349647:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.349659:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.349662:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.349664:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.349674:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.349677:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.349679:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.349683:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349690:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.349698:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.349700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.349704:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.349707:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24497, 24497] 00000080:00200000:3.0:1775764896.349718:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.349722:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.349751:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100343808 i_blocks 90112 00000080:00200000:3.0:1775764896.349755:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.349771:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100343808, count: 4096 00000080:00200000:3.0:1775764896.349781:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100343808, bytes: 4096 00000080:00200000:3.0:1775764896.349784:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.349788:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24499, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.349790:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100343808, 100347904) 0 0 00000080:00200000:3.0:1775764896.349792:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.349798:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24498, 24498] 00020000:00200000:3.0:1775764896.349800:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100343808 chunk: [100343808, 100347904) 100347904, 4096 00020000:00200000:3.0:1775764896.349803:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.349814:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24498, 24498] 00000008:00010000:3.0:1775764896.349820:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.349822:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.349830:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349839:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100343808 100347903) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349847:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349854:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.349861:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.349888:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100343808, 100347904) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.349895:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24498 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.349912:0:18302:0:(rw26.c:858:ll_write_end()) pos 100343808, len 4096, copied 4096 00000080:00200000:3.0:1775764896.349916:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007a092609[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.349919:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004add3129 17ffffc0001001 3:0 ffff90c212825c40 24498 lru 00000080:00200000:3.0:1775764896.349924:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c3091b88 24498: 1< 0 - - > 2< 100343808 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 270336 - | + - + - > 5< + - + - | 0 - | 4018 - -> 00000080:00200000:3.0:1775764896.349927:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007a092609 00000080:00200000:3.0:1775764896.349928:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.349930:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.349938:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349946:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349954:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.349961:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.349972:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.349975:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.349977:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.349986:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.349989:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.349992:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.349996:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350004:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.350012:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.350015:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.350019:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.350022:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24498, 24498] 00000080:00200000:3.0:1775764896.350032:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.350035:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.350063:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100347904 i_blocks 90112 00000080:00200000:3.0:1775764896.350067:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.350084:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100347904, count: 4096 00000080:00200000:3.0:1775764896.350095:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100347904, bytes: 4096 00000080:00200000:3.0:1775764896.350097:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.350101:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24500, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.350102:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100347904, 100352000) 0 0 00000080:00200000:3.0:1775764896.350105:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.350110:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24499, 24499] 00020000:00200000:3.0:1775764896.350419:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100347904 chunk: [100347904, 100352000) 100352000, 4096 00020000:00200000:3.0:1775764896.350422:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.350437:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24499, 24499] 00000008:00010000:3.0:1775764896.350443:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.350446:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.350454:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350463:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100347904 100351999) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350472:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350480:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350486:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.350495:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100347904, 100352000) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.350500:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24499 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.350519:0:18302:0:(rw26.c:858:ll_write_end()) pos 100347904, len 4096, copied 4096 00000080:00200000:3.0:1775764896.350523:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b1c3b4aa[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.350527:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004c5e0bcc 17ffffc0001001 3:0 ffff90c2128258c0 24499 lru 00000080:00200000:3.0:1775764896.350532:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f4ba05a0 24499: 1< 0 - - > 2< 100347904 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 266240 - | + - + - > 5< + - + - | 0 - | 4019 - -> 00000080:00200000:3.0:1775764896.350535:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b1c3b4aa 00000080:00200000:3.0:1775764896.350536:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.350538:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.350546:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.350554:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.350560:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.350574:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.350587:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.350590:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.350592:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.350603:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.350606:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.350608:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.350612:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350620:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.350628:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.350630:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.350633:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.350636:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24499, 24499] 00000080:00200000:3.0:1775764896.350647:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.350651:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.350682:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100352000 i_blocks 90112 00000080:00200000:3.0:1775764896.350686:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.350703:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100352000, count: 4096 00000080:00200000:3.0:1775764896.350715:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100352000, bytes: 4096 00000080:00200000:3.0:1775764896.350718:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.350723:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24501, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.350725:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100352000, 100356096) 0 0 00000080:00200000:3.0:1775764896.350728:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.350733:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24500, 24500] 00020000:00200000:3.0:1775764896.350736:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100352000 chunk: [100352000, 100356096) 100356096, 4096 00020000:00200000:3.0:1775764896.350739:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.350751:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24500, 24500] 00000008:00010000:3.0:1775764896.350757:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.350759:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.350767:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350776:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100352000 100356095) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350784:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350790:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.350796:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.350803:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100352000, 100356096) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.350809:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24500 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.350825:0:18302:0:(rw26.c:858:ll_write_end()) pos 100352000, len 4096, copied 4096 00000080:00200000:3.0:1775764896.350828:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c03cc85a[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.350831:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000e0b4444d 17ffffc0001001 3:0 ffff90c212825b60 24500 lru 00000080:00200000:3.0:1775764896.350836:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003a179b91 24500: 1< 0 - - > 2< 100352000 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 262144 - | + - + - > 5< + - + - | 0 - | 4020 - -> 00000080:00200000:3.0:1775764896.350839:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c03cc85a 00000080:00200000:3.0:1775764896.350840:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.350842:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.350851:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.350858:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.350864:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.350982:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.350996:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.350999:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.351002:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.351011:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.351014:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.351016:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.351020:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351028:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.351041:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.351043:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.351047:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.351051:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24500, 24500] 00000080:00200000:3.0:1775764896.351062:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.351065:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.351096:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100356096 i_blocks 90112 00000080:00200000:3.0:1775764896.351100:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.351127:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100356096, count: 4096 00000080:00200000:3.0:1775764896.351138:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100356096, bytes: 4096 00000080:00200000:3.0:1775764896.351141:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.351145:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24502, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.351147:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100356096, 100360192) 0 0 00000080:00200000:3.0:1775764896.351150:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.351155:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24501, 24501] 00020000:00200000:3.0:1775764896.351159:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100356096 chunk: [100356096, 100360192) 100360192, 4096 00020000:00200000:3.0:1775764896.351161:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.351172:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24501, 24501] 00000008:00010000:3.0:1775764896.351178:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.351180:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.351187:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351197:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100356096 100360191) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351205:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351217:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351224:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.351232:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100356096, 100360192) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.351237:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24501 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.351334:0:18302:0:(rw26.c:858:ll_write_end()) pos 100356096, len 4096, copied 4096 00000080:00200000:3.0:1775764896.351340:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000077c93a50[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.351343:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000065f0b044 17ffffc0001001 3:0 ffff90c212825000 24501 lru 00000080:00200000:3.0:1775764896.351349:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000858413e7 24501: 1< 0 - - > 2< 100356096 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 258048 - | + - + - > 5< + - + - | 0 - | 4021 - -> 00000080:00200000:3.0:1775764896.351351:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000077c93a50 00000080:00200000:3.0:1775764896.351352:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.351354:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.351363:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.351370:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.351378:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.351386:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.351404:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.351408:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.351410:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.351421:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.351424:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.351426:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.351686:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351696:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.351706:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.351709:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.351715:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.351719:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24501, 24501] 00000080:00200000:3.0:1775764896.351732:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.351735:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.351770:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100360192 i_blocks 90112 00000080:00200000:3.0:1775764896.351774:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.351792:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100360192, count: 4096 00000080:00200000:3.0:1775764896.351803:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100360192, bytes: 4096 00000080:00200000:3.0:1775764896.351806:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.351810:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24503, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.351811:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100360192, 100364288) 0 0 00000080:00200000:3.0:1775764896.351814:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.351821:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24502, 24502] 00020000:00200000:3.0:1775764896.351824:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100360192 chunk: [100360192, 100364288) 100364288, 4096 00020000:00200000:3.0:1775764896.351827:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.351839:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24502, 24502] 00000008:00010000:3.0:1775764896.351845:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.351848:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.351857:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351866:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100360192 100364287) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351898:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351906:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.351913:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.351921:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100360192, 100364288) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.351927:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24502 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.351946:0:18302:0:(rw26.c:858:ll_write_end()) pos 100360192, len 4096, copied 4096 00000080:00200000:3.0:1775764896.351951:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003a25b9aa[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.351953:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000082bb2693 17ffffc0001001 3:0 ffff90c212825ee0 24502 lru 00000080:00200000:3.0:1775764896.351958:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000d0f07101 24502: 1< 0 - - > 2< 100360192 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 253952 - | + - + - > 5< + - + - | 0 - | 4022 - -> 00000080:00200000:3.0:1775764896.351960:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003a25b9aa 00000080:00200000:3.0:1775764896.351961:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.351963:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.351972:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.351980:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.351987:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.351996:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.352008:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.352011:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.352013:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.352024:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.352027:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.352029:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.352033:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352040:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.352048:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.352051:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.352055:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.352058:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24502, 24502] 00000080:00200000:3.0:1775764896.352069:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.352072:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.352093:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100364288 i_blocks 90112 00000080:00200000:3.0:1775764896.352096:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.352107:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100364288, count: 4096 00000080:00200000:3.0:1775764896.352152:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100364288, bytes: 4096 00000080:00200000:3.0:1775764896.352156:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.352159:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24504, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.352160:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100364288, 100368384) 0 0 00000080:00200000:3.0:1775764896.352162:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.352166:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24503, 24503] 00020000:00200000:3.0:1775764896.352168:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100364288 chunk: [100364288, 100368384) 100368384, 4096 00020000:00200000:3.0:1775764896.352169:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.352177:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24503, 24503] 00000008:00010000:3.0:1775764896.352181:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.352183:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.352189:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352195:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100364288 100368383) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352201:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352206:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352209:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.352215:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100364288, 100368384) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.352219:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24503 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.352232:0:18302:0:(rw26.c:858:ll_write_end()) pos 100364288, len 4096, copied 4096 00000080:00200000:3.0:1775764896.352235:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000645671e9[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.352237:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c928acdf 17ffffc0001001 3:0 ffff90c2128250e0 24503 lru 00000080:00200000:3.0:1775764896.352241:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a574f92e 24503: 1< 0 - - > 2< 100364288 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 249856 - | + - + - > 5< + - + - | 0 - | 4023 - -> 00000080:00200000:3.0:1775764896.352242:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000645671e9 00000080:00200000:3.0:1775764896.352243:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.352245:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.352252:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.352259:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.352266:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.352272:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.352284:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.352287:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.352290:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.352300:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.352302:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.352303:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.352306:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352312:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.352317:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.352318:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.352321:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.352322:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24503, 24503] 00000080:00200000:3.0:1775764896.352330:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.352332:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.352352:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100368384 i_blocks 90112 00000080:00200000:3.0:1775764896.352354:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.352365:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100368384, count: 4096 00000080:00200000:3.0:1775764896.352371:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100368384, bytes: 4096 00000080:00200000:3.0:1775764896.352373:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.352376:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24505, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.352377:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100368384, 100372480) 0 0 00000080:00200000:3.0:1775764896.352379:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.352383:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24504, 24504] 00020000:00200000:3.0:1775764896.352385:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100368384 chunk: [100368384, 100372480) 100372480, 4096 00020000:00200000:3.0:1775764896.352386:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.352393:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24504, 24504] 00000008:00010000:3.0:1775764896.352396:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.352398:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.352403:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352409:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100368384 100372479) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352415:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352420:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352423:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.352428:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100368384, 100372480) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.352431:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24504 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.352447:0:18302:0:(rw26.c:858:ll_write_end()) pos 100368384, len 4096, copied 4096 00000080:00200000:3.0:1775764896.352450:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f5f21ac9[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.352452:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000929b5f20 17ffffc0001001 3:0 ffff90c2128251c0 24504 lru 00000080:00200000:3.0:1775764896.352456:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000bf6d4926 24504: 1< 0 - - > 2< 100368384 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 245760 - | + - + - > 5< + - + - | 0 - | 4024 - -> 00000080:00200000:3.0:1775764896.352457:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f5f21ac9 00000080:00200000:3.0:1775764896.352458:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.352459:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.352464:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.352469:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.352566:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.352578:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.352590:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.352593:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.352596:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.352606:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.352609:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.352611:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.352616:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352625:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.352633:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.352636:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.352640:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.352643:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24504, 24504] 00000080:00200000:3.0:1775764896.352654:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.352657:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.352688:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100372480 i_blocks 90112 00000080:00200000:3.0:1775764896.352691:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.352707:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100372480, count: 4096 00000080:00200000:3.0:1775764896.352718:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100372480, bytes: 4096 00000080:00200000:3.0:1775764896.352721:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.352725:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24506, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.352727:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100372480, 100376576) 0 0 00000080:00200000:3.0:1775764896.352729:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.352735:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24505, 24505] 00020000:00200000:3.0:1775764896.352737:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100372480 chunk: [100372480, 100376576) 100376576, 4096 00020000:00200000:3.0:1775764896.352740:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.352750:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24505, 24505] 00000008:00010000:3.0:1775764896.352756:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.352758:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.352766:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352775:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100372480 100376575) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352783:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352791:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.352797:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.352804:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100372480, 100376576) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.352809:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24505 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.352837:0:18302:0:(rw26.c:858:ll_write_end()) pos 100372480, len 4096, copied 4096 00000080:00200000:3.0:1775764896.352841:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000367eb26c[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.352844:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003f8dab3d 17ffffc0001001 3:0 ffff90c2128261c0 24505 lru 00000080:00200000:3.0:1775764896.352850:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003d93998d 24505: 1< 0 - - > 2< 100372480 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 241664 - | + - + - > 5< + - + - | 0 - | 4025 - -> 00000080:00200000:3.0:1775764896.352852:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000367eb26c 00000080:00200000:3.0:1775764896.352853:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.352855:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.352864:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.352896:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.352915:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.352923:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.353084:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.353088:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.353091:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.353102:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.353104:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.353107:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.353121:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353130:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.353145:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.353148:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.353153:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.353156:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24505, 24505] 00000080:00200000:3.0:1775764896.353167:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.353171:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.353202:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100376576 i_blocks 90112 00000080:00200000:3.0:1775764896.353206:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.353223:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100376576, count: 4096 00000080:00200000:3.0:1775764896.353232:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100376576, bytes: 4096 00000080:00200000:3.0:1775764896.353235:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.353239:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24507, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.353241:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100376576, 100380672) 0 0 00000080:00200000:3.0:1775764896.353243:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.353249:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24506, 24506] 00020000:00200000:3.0:1775764896.353252:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100376576 chunk: [100376576, 100380672) 100380672, 4096 00020000:00200000:3.0:1775764896.353255:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.353267:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24506, 24506] 00000008:00010000:3.0:1775764896.353273:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.353276:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.353284:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353293:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100376576 100380671) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353301:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353313:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353320:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.353328:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100376576, 100380672) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.353334:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24506 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.353352:0:18302:0:(rw26.c:858:ll_write_end()) pos 100376576, len 4096, copied 4096 00000080:00200000:3.0:1775764896.353357:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c9785484[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.353360:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000deeff769 17ffffc0001001 3:0 ffff90c2128267e0 24506 lru 00000080:00200000:3.0:1775764896.353366:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000920661b1 24506: 1< 0 - - > 2< 100376576 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 237568 - | + - + - > 5< + - + - | 0 - | 4026 - -> 00000080:00200000:3.0:1775764896.353368:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c9785484 00000080:00200000:3.0:1775764896.353369:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.353371:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.353380:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.353389:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.353396:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.353404:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.353422:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.353425:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.353428:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.353438:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.353441:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.353443:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.353447:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353455:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.353463:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.353466:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.353550:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.353555:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24506, 24506] 00000080:00200000:3.0:1775764896.353566:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.353570:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.353601:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100380672 i_blocks 90112 00000080:00200000:3.0:1775764896.353605:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.353621:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100380672, count: 4096 00000080:00200000:3.0:1775764896.353632:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100380672, bytes: 4096 00000080:00200000:3.0:1775764896.353635:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.353639:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24508, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.353641:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100380672, 100384768) 0 0 00000080:00200000:3.0:1775764896.353643:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.353649:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24507, 24507] 00020000:00200000:3.0:1775764896.353653:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100380672 chunk: [100380672, 100384768) 100384768, 4096 00020000:00200000:3.0:1775764896.353655:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.353667:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24507, 24507] 00000008:00010000:3.0:1775764896.353673:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.353675:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.353684:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353693:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100380672 100384767) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353703:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353711:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353718:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.353727:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100380672, 100384768) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.353732:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24507 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.353751:0:18302:0:(rw26.c:858:ll_write_end()) pos 100380672, len 4096, copied 4096 00000080:00200000:3.0:1775764896.353755:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000003f29686[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.353758:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006d8632c3 17ffffc0001001 3:0 ffff90c2128269a0 24507 lru 00000080:00200000:3.0:1775764896.353765:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000be04dde1 24507: 1< 0 - - > 2< 100380672 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 233472 - | + - + - > 5< + - + - | 0 - | 4027 - -> 00000080:00200000:3.0:1775764896.353767:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000003f29686 00000080:00200000:3.0:1775764896.353768:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.353771:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.353778:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.353787:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.353794:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.353802:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.353813:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.353816:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.353818:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.353828:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.353830:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.353833:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.353837:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.353846:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.353854:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.353857:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.353861:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.353864:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24507, 24507] 00000080:00200000:3.0:1775764896.354019:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.354024:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.354056:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100384768 i_blocks 90112 00000080:00200000:3.0:1775764896.354060:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.354078:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100384768, count: 4096 00000080:00200000:3.0:1775764896.354089:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100384768, bytes: 4096 00000080:00200000:3.0:1775764896.354092:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.354096:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24509, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.354098:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100384768, 100388864) 0 0 00000080:00200000:3.0:1775764896.354100:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.354106:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24508, 24508] 00020000:00200000:3.0:1775764896.354109:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100384768 chunk: [100384768, 100388864) 100388864, 4096 00020000:00200000:3.0:1775764896.354121:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.354134:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24508, 24508] 00000008:00010000:3.0:1775764896.354139:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.354141:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.354149:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354159:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100384768 100388863) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354168:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354176:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354185:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.354193:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100384768, 100388864) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.354199:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24508 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.354217:0:18302:0:(rw26.c:858:ll_write_end()) pos 100384768, len 4096, copied 4096 00000080:00200000:3.0:1775764896.354221:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000705a8e7b[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.354224:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000096497495 17ffffc0001001 3:0 ffff90c2128262a0 24508 lru 00000080:00200000:3.0:1775764896.354230:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e397449e 24508: 1< 0 - - > 2< 100384768 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 229376 - | + - + - > 5< + - + - | 0 - | 4028 - -> 00000080:00200000:3.0:1775764896.354232:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000705a8e7b 00000080:00200000:3.0:1775764896.354234:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.354236:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.354244:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.354253:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.354260:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.354268:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.354279:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.354282:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.354284:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.354295:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.354298:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.354300:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.354304:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354312:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.354319:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.354321:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.354325:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.354328:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24508, 24508] 00000080:00200000:3.0:1775764896.354338:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.354342:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.354372:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100388864 i_blocks 90112 00000080:00200000:3.0:1775764896.354375:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.354392:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100388864, count: 4096 00000080:00200000:3.0:1775764896.354402:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100388864, bytes: 4096 00000080:00200000:3.0:1775764896.354405:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.354409:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24510, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.354410:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100388864, 100392960) 0 0 00000080:00200000:3.0:1775764896.354412:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.354418:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24509, 24509] 00020000:00200000:3.0:1775764896.354421:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100388864 chunk: [100388864, 100392960) 100392960, 4096 00020000:00200000:3.0:1775764896.354423:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.354435:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24509, 24509] 00000008:00010000:3.0:1775764896.354440:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.354443:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.354451:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354462:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100388864 100392959) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354600:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354611:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354619:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.354628:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100388864, 100392960) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.354634:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24509 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.354651:0:18302:0:(rw26.c:858:ll_write_end()) pos 100388864, len 4096, copied 4096 00000080:00200000:3.0:1775764896.354655:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000c618083[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.354658:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000aef5d462 17ffffc0001001 3:0 ffff90c212826d20 24509 lru 00000080:00200000:3.0:1775764896.354664:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009c65bf4f 24509: 1< 0 - - > 2< 100388864 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 225280 - | + - + - > 5< + - + - | 0 - | 4029 - -> 00000080:00200000:3.0:1775764896.354667:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000c618083 00000080:00200000:3.0:1775764896.354668:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.354670:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.354679:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.354687:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.354695:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.354709:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.354723:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.354726:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.354728:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.354739:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.354742:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.354744:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.354749:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.354757:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.354766:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.354768:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.354773:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.354776:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24509, 24509] 00000080:00200000:3.0:1775764896.354787:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.354790:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.354820:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100392960 i_blocks 90112 00000080:00200000:3.0:1775764896.354823:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.354841:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100392960, count: 4096 00000080:00200000:3.0:1775764896.354852:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100392960, bytes: 4096 00000080:00200000:3.0:1775764896.354855:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.354859:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24511, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.354861:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100392960, 100397056) 0 0 00000080:00200000:3.0:1775764896.354863:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.354891:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24510, 24510] 00020000:00200000:3.0:1775764896.354895:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100392960 chunk: [100392960, 100397056) 100397056, 4096 00020000:00200000:3.0:1775764896.355039:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.355055:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24510, 24510] 00000008:00010000:3.0:1775764896.355061:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.355064:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.355073:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355083:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100392960 100397055) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355092:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355100:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355108:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.355127:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100392960, 100397056) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.355133:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24510 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.355151:0:18302:0:(rw26.c:858:ll_write_end()) pos 100392960, len 4096, copied 4096 00000080:00200000:3.0:1775764896.355156:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000de8c3675[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.355159:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000021e71ec1 17ffffc0001001 3:0 ffff90c212826380 24510 lru 00000080:00200000:3.0:1775764896.355165:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f329e3cd 24510: 1< 0 - - > 2< 100392960 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 221184 - | + - + - > 5< + - + - | 0 - | 4030 - -> 00000080:00200000:3.0:1775764896.355167:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000de8c3675 00000080:00200000:3.0:1775764896.355168:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.355170:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.355178:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.355187:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.355194:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.355202:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.355214:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.355218:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.355220:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.355231:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.355233:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.355236:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.355240:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355248:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.355264:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.355266:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.355270:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.355273:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24510, 24510] 00000080:00200000:3.0:1775764896.355284:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.355287:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.355316:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100397056 i_blocks 90112 00000080:00200000:3.0:1775764896.355320:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.355337:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100397056, count: 4096 00000080:00200000:3.0:1775764896.355349:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100397056, bytes: 4096 00000080:00200000:3.0:1775764896.355351:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.355355:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24512, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.355357:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100397056, 100401152) 0 0 00000080:00200000:3.0:1775764896.355359:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.355364:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24511, 24511] 00020000:00200000:3.0:1775764896.355368:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100397056 chunk: [100397056, 100401152) 100401152, 4096 00020000:00200000:3.0:1775764896.355370:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.355382:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24511, 24511] 00000008:00010000:3.0:1775764896.355388:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.355391:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.355399:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355409:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100397056 100401151) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355417:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355430:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355437:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.355444:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100397056, 100401152) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.355450:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24511 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.355550:0:18302:0:(rw26.c:858:ll_write_end()) pos 100397056, len 4096, copied 4096 00000080:00200000:3.0:1775764896.355556:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c3ca5b5f[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.355559:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000f2c4b30 17ffffc0001021 2:0 ffff90c212826620 24511 lru 00000080:00200000:3.0:1775764896.355566:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000467757af 24511: 1< 0 - - > 2< 100397056 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 217088 - | + - + - > 5< + - + - | 0 - | 4031 - -> 00000080:00200000:3.0:1775764896.355568:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c3ca5b5f 00000080:00200000:3.0:1775764896.355569:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.355571:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.355580:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.355588:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.355596:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.355604:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.355621:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.355624:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.355627:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.355637:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.355640:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.355643:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.355647:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355655:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.355664:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.355666:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.355671:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.355673:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24511, 24511] 00000080:00200000:3.0:1775764896.355685:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.355688:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.355718:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100401152 i_blocks 90112 00000080:00200000:3.0:1775764896.355722:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.355738:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100401152, count: 4096 00000080:00200000:3.0:1775764896.355749:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100401152, bytes: 4096 00000080:00200000:3.0:1775764896.355752:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.355756:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24513, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.355758:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100401152, 100405248) 0 0 00000080:00200000:3.0:1775764896.355760:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.355765:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24512, 24512] 00020000:00200000:3.0:1775764896.355768:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100401152 chunk: [100401152, 100405248) 100405248, 4096 00020000:00200000:3.0:1775764896.355771:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.355782:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24512, 24512] 00000008:00010000:3.0:1775764896.355788:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.355790:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.355798:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355807:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100401152 100405247) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355816:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355824:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.355832:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.355840:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100401152, 100405248) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.355846:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24512 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.355868:0:18302:0:(rw26.c:858:ll_write_end()) pos 100401152, len 4096, copied 4096 00000080:00200000:3.0:1775764896.355894:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e25b914b[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.355897:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008fcdd43e 17ffffc0001001 3:0 ffff90c212826e00 24512 lru 00000080:00200000:3.0:1775764896.356061:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000013895b92 24512: 1< 0 - - > 2< 100401152 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 212992 - | + - + - > 5< + - + - | 0 - | 4032 - -> 00000080:00200000:3.0:1775764896.356064:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e25b914b 00000080:00200000:3.0:1775764896.356065:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.356068:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.356078:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.356087:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.356094:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.356101:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.356124:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.356128:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.356130:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.356142:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.356145:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.356147:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.356151:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356160:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.356168:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.356171:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.356175:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.356178:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24512, 24512] 00000080:00200000:3.0:1775764896.356189:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.356192:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.356222:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100405248 i_blocks 90112 00000080:00200000:3.0:1775764896.356226:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.356238:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100405248, count: 4096 00000080:00200000:3.0:1775764896.356246:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100405248, bytes: 4096 00000080:00200000:3.0:1775764896.356249:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.356252:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24514, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.356254:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100405248, 100409344) 0 0 00000080:00200000:3.0:1775764896.356257:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.356262:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24513, 24513] 00020000:00200000:3.0:1775764896.356265:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100405248 chunk: [100405248, 100409344) 100409344, 4096 00020000:00200000:3.0:1775764896.356268:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.356280:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24513, 24513] 00000008:00010000:3.0:1775764896.356286:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.356289:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.356296:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356305:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100405248 100409343) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356313:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356320:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356327:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.356334:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100405248, 100409344) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.356339:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24513 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.356358:0:18302:0:(rw26.c:858:ll_write_end()) pos 100405248, len 4096, copied 4096 00000080:00200000:3.0:1775764896.356362:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b2ddf958[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.356365:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000ed982d8a 17ffffc0001001 3:0 ffff90c212826540 24513 lru 00000080:00200000:3.0:1775764896.356369:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e7981124 24513: 1< 0 - - > 2< 100405248 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 208896 - | + - + - > 5< + - + - | 0 - | 4033 - -> 00000080:00200000:3.0:1775764896.356371:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b2ddf958 00000080:00200000:3.0:1775764896.356371:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.356373:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.356379:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.356384:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.356388:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.356392:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.356399:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.356401:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.356402:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.356409:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.356411:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.356412:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.356415:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356420:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.356425:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.356427:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.356429:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.356431:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24513, 24513] 00000080:00200000:3.0:1775764896.356439:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.356441:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.356460:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100409344 i_blocks 90112 00000080:00200000:3.0:1775764896.356462:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.356572:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100409344, count: 4096 00000080:00200000:3.0:1775764896.356581:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100409344, bytes: 4096 00000080:00200000:3.0:1775764896.356584:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.356586:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24515, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.356587:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100409344, 100413440) 0 0 00000080:00200000:3.0:1775764896.356589:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.356593:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24514, 24514] 00020000:00200000:3.0:1775764896.356596:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100409344 chunk: [100409344, 100413440) 100413440, 4096 00020000:00200000:3.0:1775764896.356597:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.356604:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24514, 24514] 00000008:00010000:3.0:1775764896.356608:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.356609:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.356615:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356623:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100409344 100413439) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356632:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356640:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356645:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.356650:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100409344, 100413440) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.356654:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24514 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.356667:0:18302:0:(rw26.c:858:ll_write_end()) pos 100409344, len 4096, copied 4096 00000080:00200000:3.0:1775764896.356670:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004637ddef[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.356672:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006965c7a0 17ffffc0001001 3:0 ffff90c212826460 24514 lru 00000080:00200000:3.0:1775764896.356676:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006627e9e8 24514: 1< 0 - - > 2< 100409344 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 204800 - | + - + - > 5< + - + - | 0 - | 4034 - -> 00000080:00200000:3.0:1775764896.356678:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004637ddef 00000080:00200000:3.0:1775764896.356678:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.356680:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.356685:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.356690:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.356694:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.356702:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.356710:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.356711:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.356713:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.356719:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.356721:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.356723:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.356725:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356730:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.356735:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.356737:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.356739:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.356741:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24514, 24514] 00000080:00200000:3.0:1775764896.356748:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.356750:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.356769:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100413440 i_blocks 90112 00000080:00200000:3.0:1775764896.356772:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.356782:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100413440, count: 4096 00000080:00200000:3.0:1775764896.356788:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100413440, bytes: 4096 00000080:00200000:3.0:1775764896.356790:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.356793:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24516, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.356794:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100413440, 100417536) 0 0 00000080:00200000:3.0:1775764896.356796:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.356799:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24515, 24515] 00020000:00200000:3.0:1775764896.356801:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100413440 chunk: [100413440, 100417536) 100417536, 4096 00020000:00200000:3.0:1775764896.356802:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.356809:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24515, 24515] 00000008:00010000:3.0:1775764896.356812:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.356814:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.356819:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356825:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100413440 100417535) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356831:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356835:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.356839:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.356844:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100413440, 100417536) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.356847:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24515 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.356859:0:18302:0:(rw26.c:858:ll_write_end()) pos 100413440, len 4096, copied 4096 00000080:00200000:3.0:1775764896.356862:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000caf60df8[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.356864:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000667066f9 17ffffc0001001 3:0 ffff90c212826a80 24515 lru 00000080:00200000:3.0:1775764896.356868:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006c8b3f64 24515: 1< 0 - - > 2< 100413440 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 200704 - | + - + - > 5< + - + - | 0 - | 4035 - -> 00000080:00200000:3.0:1775764896.356999:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000caf60df8 00000080:00200000:3.0:1775764896.357000:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.357003:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.357012:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.357020:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.357026:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.357032:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.357043:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.357046:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.357048:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.357057:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.357059:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.357061:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.357065:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357073:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.357087:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.357090:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.357094:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.357097:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24515, 24515] 00000080:00200000:3.0:1775764896.357107:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.357110:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.357147:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100417536 i_blocks 90112 00000080:00200000:3.0:1775764896.357151:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.357167:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100417536, count: 4096 00000080:00200000:3.0:1775764896.357177:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100417536, bytes: 4096 00000080:00200000:3.0:1775764896.357180:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.357184:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24517, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.357186:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100417536, 100421632) 0 0 00000080:00200000:3.0:1775764896.357188:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.357193:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24516, 24516] 00020000:00200000:3.0:1775764896.357197:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100417536 chunk: [100417536, 100421632) 100421632, 4096 00020000:00200000:3.0:1775764896.357199:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.357210:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24516, 24516] 00000008:00010000:3.0:1775764896.357215:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.357217:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.357225:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357234:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100417536 100421631) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357242:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357254:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357261:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.357268:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100417536, 100421632) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.357274:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24516 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.357292:0:18302:0:(rw26.c:858:ll_write_end()) pos 100417536, len 4096, copied 4096 00000080:00200000:3.0:1775764896.357297:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000486b558c[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.357299:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b5df721d 17ffffc0001001 3:0 ffff90c212826700 24516 lru 00000080:00200000:3.0:1775764896.357304:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000556ce33f 24516: 1< 0 - - > 2< 100417536 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 196608 - | + - + - > 5< + - + - | 0 - | 4036 - -> 00000080:00200000:3.0:1775764896.357306:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000486b558c 00000080:00200000:3.0:1775764896.357307:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.357309:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.357316:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.357323:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.357329:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.357337:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.357354:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.357356:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.357358:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.357368:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.357371:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.357373:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.357377:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357384:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.357391:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.357393:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.357396:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.357399:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24516, 24516] 00000080:00200000:3.0:1775764896.357409:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.357412:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.357442:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100421632 i_blocks 90112 00000080:00200000:3.0:1775764896.357446:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.357463:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100421632, count: 4096 00000080:00200000:3.0:1775764896.357554:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100421632, bytes: 4096 00000080:00200000:3.0:1775764896.357557:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.357561:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24518, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.357562:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100421632, 100425728) 0 0 00000080:00200000:3.0:1775764896.357565:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.357571:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24517, 24517] 00020000:00200000:3.0:1775764896.357574:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100421632 chunk: [100421632, 100425728) 100425728, 4096 00020000:00200000:3.0:1775764896.357576:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.357589:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24517, 24517] 00000008:00010000:3.0:1775764896.357594:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.357597:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.357605:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357613:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100421632 100425727) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357622:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357629:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357636:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.357644:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100421632, 100425728) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.357650:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24517 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.357667:0:18302:0:(rw26.c:858:ll_write_end()) pos 100421632, len 4096, copied 4096 00000080:00200000:3.0:1775764896.357671:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000260f3acf[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.357674:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000cf90a66 17ffffc0001001 3:0 ffff90c212826c40 24517 lru 00000080:00200000:3.0:1775764896.357679:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002bdfe9a7 24517: 1< 0 - - > 2< 100421632 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 192512 - | + - + - > 5< + - + - | 0 - | 4037 - -> 00000080:00200000:3.0:1775764896.357681:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000260f3acf 00000080:00200000:3.0:1775764896.357682:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.357684:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.357693:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.357700:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.357706:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.357714:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.357726:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.357729:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.357731:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.357741:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.357744:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.357746:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.357750:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.357758:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.357766:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.357768:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.357773:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.357776:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24517, 24517] 00000080:00200000:3.0:1775764896.357787:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.357790:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.357820:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100425728 i_blocks 90112 00000080:00200000:3.0:1775764896.357824:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.357840:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100425728, count: 4096 00000080:00200000:3.0:1775764896.357851:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100425728, bytes: 4096 00000080:00200000:3.0:1775764896.357853:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.357857:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24519, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.357858:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100425728, 100429824) 0 0 00000080:00200000:3.0:1775764896.357861:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.357867:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24518, 24518] 00020000:00200000:3.0:1775764896.357889:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100425728 chunk: [100425728, 100429824) 100429824, 4096 00020000:00200000:3.0:1775764896.357892:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.357991:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24518, 24518] 00000008:00010000:3.0:1775764896.357998:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.358000:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.358008:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358018:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100425728 100429823) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358026:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358033:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358040:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.358048:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100425728, 100429824) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.358053:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24518 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.358071:0:18302:0:(rw26.c:858:ll_write_end()) pos 100425728, len 4096, copied 4096 00000080:00200000:3.0:1775764896.358075:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000a31fcf0[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.358078:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000017bfa0e6 17ffffc0001001 3:0 ffff90c2128268c0 24518 lru 00000080:00200000:3.0:1775764896.358083:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b37c7149 24518: 1< 0 - - > 2< 100425728 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 188416 - | + - + - > 5< + - + - | 0 - | 4038 - -> 00000080:00200000:3.0:1775764896.358085:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000a31fcf0 00000080:00200000:3.0:1775764896.358086:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.358089:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.358098:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.358105:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.358124:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.358132:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.358145:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.358148:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.358151:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.358161:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.358164:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.358166:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.358171:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358178:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.358186:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.358189:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.358193:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.358196:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24518, 24518] 00000080:00200000:3.0:1775764896.358206:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.358210:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.358241:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100429824 i_blocks 90112 00000080:00200000:3.0:1775764896.358245:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.358261:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100429824, count: 4096 00000080:00200000:3.0:1775764896.358273:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100429824, bytes: 4096 00000080:00200000:3.0:1775764896.358276:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.358280:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24520, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.358282:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100429824, 100433920) 0 0 00000080:00200000:3.0:1775764896.358284:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.358290:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24519, 24519] 00020000:00200000:3.0:1775764896.358293:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100429824 chunk: [100429824, 100433920) 100433920, 4096 00020000:00200000:3.0:1775764896.358296:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.358308:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24519, 24519] 00000008:00010000:3.0:1775764896.358314:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.358316:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.358324:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358332:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100429824 100433919) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358341:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358348:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358355:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.358363:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100429824, 100433920) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.358368:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24519 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.358387:0:18302:0:(rw26.c:858:ll_write_end()) pos 100429824, len 4096, copied 4096 00000080:00200000:3.0:1775764896.358391:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000089479067[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.358394:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000a44b12d 17ffffc0001001 3:0 ffff90c212826b60 24519 lru 00000080:00200000:3.0:1775764896.358399:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000015c06815 24519: 1< 0 - - > 2< 100429824 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 184320 - | + - + - > 5< + - + - | 0 - | 4039 - -> 00000080:00200000:3.0:1775764896.358401:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000089479067 00000080:00200000:3.0:1775764896.358402:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.358405:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.358413:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.358420:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.358427:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.358439:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.358450:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.358453:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.358455:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.358466:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.358468:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.358553:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.358560:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358567:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.358576:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.358579:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.358584:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.358587:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24519, 24519] 00000080:00200000:3.0:1775764896.358599:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.358602:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:3.0:1775764896.358634:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100433920 i_blocks 90112 00000080:00200000:3.0:1775764896.358638:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:3.0:1775764896.358655:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100433920, count: 4096 00000080:00200000:3.0:1775764896.358666:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100433920, bytes: 4096 00000080:00200000:3.0:1775764896.358669:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:3.0:1775764896.358673:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24521, [0x200000403:0x10:0x0] hash] 00000020:00200000:3.0:1775764896.358675:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100433920, 100438016) 0 0 00000080:00200000:3.0:1775764896.358677:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:3.0:1775764896.358683:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24520, 24520] 00020000:00200000:3.0:1775764896.358686:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100433920 chunk: [100433920, 100438016) 100438016, 4096 00020000:00200000:3.0:1775764896.358689:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:3.0:1775764896.358701:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24520, 24520] 00000008:00010000:3.0:1775764896.358706:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:3.0:1775764896.358709:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c20ac7eeb0, osc lock ffff90c2126d6d10, flags 0x0 00010000:00010000:3.0:1775764896.358717:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358725:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100433920 100438015) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358733:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358741:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358748:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.358756:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100433920, 100438016) append 0 append lockpos 0 00000080:00200000:3.0:1775764896.358762:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24520 of 0 to 4096 bytes 00000080:00200000:3.0:1775764896.358778:0:18302:0:(rw26.c:858:ll_write_end()) pos 100433920, len 4096, copied 4096 00000080:00200000:3.0:1775764896.358782:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000098e50458[3 00000000b99c1f5d 2 1 0000000070fbe4a7] 00000080:00200000:3.0:1775764896.358784:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002fee60a6 17ffffc0001001 3:0 ffff90c212826000 24520 lru 00000080:00200000:3.0:1775764896.358790:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006e621fba 24520: 1< 0 - - > 2< 100433920 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 180224 - | + - + - > 5< + - + - | 0 - | 4040 - -> 00000080:00200000:3.0:1775764896.358792:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000098e50458 00000080:00200000:3.0:1775764896.358793:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:3.0:1775764896.358795:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:3.0:1775764896.358803:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.358810:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 00000000c8a2037a ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.358817:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:3.0:1775764896.358824:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:3.0:1775764896.358834:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:3.0:1775764896.358837:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:3.0:1775764896.358839:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:3.0:1775764896.358849:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_release():245 00000020:00010000:3.0:1775764896.358852:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_cancel():142 00000020:00010000:3.0:1775764896.358854:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c20ac7eeb0 (ffff90c203cb9098/1) at cl_lock_cancel():142 00010000:00010000:3.0:1775764896.358858:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:3.0:1775764896.358866:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:3.0:1775764896.359070:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20a614998 (ffff90c203cb9c68/0) at cl_lock_fini():72 00000020:00010000:3.0:1775764896.359073:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c20ac7eeb0 (ffff90c203cb9c68/1) at cl_lock_fini():72 00000080:00200000:3.0:1775764896.359078:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:3.0:1775764896.359081:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24520, 24520] 00000080:00200000:3.0:1775764896.359092:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:3.0:1775764896.359095:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.359213:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100438016 i_blocks 90112 00000080:00200000:0.0:1775764896.359218:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.359240:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100438016, count: 4096 00000080:00200000:0.0:1775764896.359258:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100438016, bytes: 4096 00000080:00200000:0.0:1775764896.359262:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.359266:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24522, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.359267:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100438016, 100442112) 0 0 00000080:00200000:0.0:1775764896.359269:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.359275:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24521, 24521] 00020000:00200000:0.0:1775764896.359279:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100438016 chunk: [100438016, 100442112) 100442112, 4096 00020000:00200000:0.0:1775764896.359281:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.359299:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24521, 24521] 00000008:00010000:0.0:1775764896.359304:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.359306:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.359313:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359319:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100438016 100442111) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359325:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359332:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359339:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.359348:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100438016, 100442112) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.359353:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24521 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.359373:0:18302:0:(rw26.c:858:ll_write_end()) pos 100438016, len 4096, copied 4096 00000080:00200000:0.0:1775764896.359377:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000984f12c8[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.359380:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a67b1812 17ffffc0001001 3:0 ffff90c212833540 24521 lru 00000080:00200000:0.0:1775764896.359384:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000147c77cd 24521: 1< 0 - - > 2< 100438016 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 176128 - | + - + - > 5< + - + - | 0 - | 4041 - -> 00000080:00200000:0.0:1775764896.359386:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000984f12c8 00000080:00200000:0.0:1775764896.359387:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.359390:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.359398:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.359406:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.359413:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.359420:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.359431:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.359434:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.359436:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.359449:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.359452:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.359454:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.359459:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359466:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.359549:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.359551:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.359556:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.359558:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24521, 24521] 00000080:00200000:0.0:1775764896.359570:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.359572:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.359595:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100442112 i_blocks 90112 00000080:00200000:0.0:1775764896.359597:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.359610:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100442112, count: 4096 00000080:00200000:0.0:1775764896.359616:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100442112, bytes: 4096 00000080:00200000:0.0:1775764896.359618:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.359621:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24523, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.359622:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100442112, 100446208) 0 0 00000080:00200000:0.0:1775764896.359623:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.359627:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24522, 24522] 00020000:00200000:0.0:1775764896.359629:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100442112 chunk: [100442112, 100446208) 100446208, 4096 00020000:00200000:0.0:1775764896.359631:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.359638:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24522, 24522] 00000008:00010000:0.0:1775764896.359642:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.359644:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.359650:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359656:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100442112 100446207) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359661:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359666:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359670:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.359675:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100442112, 100446208) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.359678:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24522 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.359691:0:18302:0:(rw26.c:858:ll_write_end()) pos 100442112, len 4096, copied 4096 00000080:00200000:0.0:1775764896.359694:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000071dd9b62[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.359696:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000eb7c86bd 17ffffc0001001 3:0 ffff90c212833460 24522 lru 00000080:00200000:0.0:1775764896.359700:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000829fcb86 24522: 1< 0 - - > 2< 100442112 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 172032 - | + - + - > 5< + - + - | 0 - | 4042 - -> 00000080:00200000:0.0:1775764896.359701:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000071dd9b62 00000080:00200000:0.0:1775764896.359702:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.359704:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.359709:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.359714:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.359718:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.359726:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.359733:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.359735:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.359736:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.359743:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.359745:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.359747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.359749:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359754:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.359759:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.359760:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.359763:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.359765:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24522, 24522] 00000080:00200000:0.0:1775764896.359771:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.359774:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.359792:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100446208 i_blocks 90112 00000080:00200000:0.0:1775764896.359794:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.359805:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100446208, count: 4096 00000080:00200000:0.0:1775764896.359811:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100446208, bytes: 4096 00000080:00200000:0.0:1775764896.359813:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.359815:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24524, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.359817:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100446208, 100450304) 0 0 00000080:00200000:0.0:1775764896.359818:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.359822:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24523, 24523] 00020000:00200000:0.0:1775764896.359824:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100446208 chunk: [100446208, 100450304) 100450304, 4096 00020000:00200000:0.0:1775764896.359825:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.359832:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24523, 24523] 00000008:00010000:0.0:1775764896.359835:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.359836:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.359841:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359847:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100446208 100450303) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359853:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359857:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359861:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.359865:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100446208, 100450304) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.359868:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24523 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.359896:0:18302:0:(rw26.c:858:ll_write_end()) pos 100446208, len 4096, copied 4096 00000080:00200000:0.0:1775764896.359908:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000bcd23088[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.359910:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a520c986 17ffffc0001001 3:0 ffff90c212833a80 24523 lru 00000080:00200000:0.0:1775764896.359914:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b1b86122 24523: 1< 0 - - > 2< 100446208 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 167936 - | + - + - > 5< + - + - | 0 - | 4043 - -> 00000080:00200000:0.0:1775764896.359915:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000bcd23088 00000080:00200000:0.0:1775764896.359916:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.359918:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.359923:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.359928:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.359932:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.359936:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.359943:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.359945:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.359946:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.359953:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.359954:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.359956:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.359958:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.359964:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.359972:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.359974:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.359976:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.359978:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24523, 24523] 00000080:00200000:0.0:1775764896.359985:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.359987:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.360005:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100450304 i_blocks 90112 00000080:00200000:0.0:1775764896.360008:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.360019:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100450304, count: 4096 00000080:00200000:0.0:1775764896.360025:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100450304, bytes: 4096 00000080:00200000:0.0:1775764896.360026:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.360029:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24525, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.360030:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100450304, 100454400) 0 0 00000080:00200000:0.0:1775764896.360032:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.360035:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24524, 24524] 00020000:00200000:0.0:1775764896.360037:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100450304 chunk: [100450304, 100454400) 100454400, 4096 00020000:00200000:0.0:1775764896.360038:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.360045:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24524, 24524] 00000008:00010000:0.0:1775764896.360048:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.360050:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.360055:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360061:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100450304 100454399) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360066:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360075:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360079:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.360083:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100450304, 100454400) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.360086:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24524 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.360099:0:18302:0:(rw26.c:858:ll_write_end()) pos 100450304, len 4096, copied 4096 00000080:00200000:0.0:1775764896.360101:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003a117d97[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.360103:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000fccef0b7 17ffffc0001001 3:0 ffff90c212833700 24524 lru 00000080:00200000:0.0:1775764896.360106:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000abddb793 24524: 1< 0 - - > 2< 100450304 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 163840 - | + - + - > 5< + - + - | 0 - | 4044 - -> 00000080:00200000:0.0:1775764896.360107:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003a117d97 00000080:00200000:0.0:1775764896.360108:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.360109:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.360128:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360133:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360138:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360142:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.360152:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.360154:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.360156:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.360163:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.360164:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.360166:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.360169:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360174:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.360179:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.360180:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.360183:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.360184:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24524, 24524] 00000080:00200000:0.0:1775764896.360191:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.360193:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.360212:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100454400 i_blocks 90112 00000080:00200000:0.0:1775764896.360214:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.360225:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100454400, count: 4096 00000080:00200000:0.0:1775764896.360230:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100454400, bytes: 4096 00000080:00200000:0.0:1775764896.360232:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.360235:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24526, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.360236:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100454400, 100458496) 0 0 00000080:00200000:0.0:1775764896.360237:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.360241:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24525, 24525] 00020000:00200000:0.0:1775764896.360244:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100454400 chunk: [100454400, 100458496) 100458496, 4096 00020000:00200000:0.0:1775764896.360246:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.360257:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24525, 24525] 00000008:00010000:0.0:1775764896.360262:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.360265:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.360273:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360281:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100454400 100458495) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360289:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360296:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360303:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.360311:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100454400, 100458496) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.360316:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24525 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.360331:0:18302:0:(rw26.c:858:ll_write_end()) pos 100454400, len 4096, copied 4096 00000080:00200000:0.0:1775764896.360334:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000006ceca5b[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.360336:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a42c33ec 17ffffc0001001 3:0 ffff90c212833c40 24525 lru 00000080:00200000:0.0:1775764896.360339:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000bf5b7f3a 24525: 1< 0 - - > 2< 100454400 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 159744 - | + - + - > 5< + - + - | 0 - | 4045 - -> 00000080:00200000:0.0:1775764896.360341:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000006ceca5b 00000080:00200000:0.0:1775764896.360341:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.360343:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.360349:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360354:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360358:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360362:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.360369:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.360370:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.360371:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.360378:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.360380:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.360382:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.360384:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360389:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.360394:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.360395:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.360398:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.360399:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24525, 24525] 00000080:00200000:0.0:1775764896.360407:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.360409:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.360427:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100458496 i_blocks 90112 00000080:00200000:0.0:1775764896.360430:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.360440:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100458496, count: 4096 00000080:00200000:0.0:1775764896.360447:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100458496, bytes: 4096 00000080:00200000:0.0:1775764896.360448:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.360451:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24527, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.360452:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100458496, 100462592) 0 0 00000080:00200000:0.0:1775764896.360453:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.360457:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24526, 24526] 00020000:00200000:0.0:1775764896.360459:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100458496 chunk: [100458496, 100462592) 100462592, 4096 00020000:00200000:0.0:1775764896.360460:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.360466:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24526, 24526] 00000008:00010000:0.0:1775764896.360470:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.360580:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.360587:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360594:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100458496 100462591) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360599:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360604:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360608:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.360614:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100458496, 100462592) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.360617:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24526 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.360630:0:18302:0:(rw26.c:858:ll_write_end()) pos 100458496, len 4096, copied 4096 00000080:00200000:0.0:1775764896.360633:0:18302:0:(rw26.c:916:ll_write_end()) page@000000004443beb1[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.360635:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000005233e28f 17ffffc0001001 3:0 ffff90c2128338c0 24526 lru 00000080:00200000:0.0:1775764896.360638:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000e1c32dc9 24526: 1< 0 - - > 2< 100458496 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 155648 - | + - + - > 5< + - + - | 0 - | 4046 - -> 00000080:00200000:0.0:1775764896.360640:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000004443beb1 00000080:00200000:0.0:1775764896.360641:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.360642:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.360647:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360652:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360656:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360660:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.360667:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.360668:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.360670:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.360676:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.360678:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.360679:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.360682:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360689:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.360698:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.360700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.360704:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.360707:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24526, 24526] 00000080:00200000:0.0:1775764896.360717:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.360720:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.360750:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100462592 i_blocks 90112 00000080:00200000:0.0:1775764896.360754:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.360769:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100462592, count: 4096 00000080:00200000:0.0:1775764896.360779:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100462592, bytes: 4096 00000080:00200000:0.0:1775764896.360782:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.360786:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24528, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.360787:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100462592, 100466688) 0 0 00000080:00200000:0.0:1775764896.360790:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.360795:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24527, 24527] 00020000:00200000:0.0:1775764896.360798:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100462592 chunk: [100462592, 100466688) 100466688, 4096 00020000:00200000:0.0:1775764896.360800:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.360811:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24527, 24527] 00000008:00010000:0.0:1775764896.360815:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.360816:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.360822:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360829:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100462592 100466687) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360835:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360839:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.360843:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.360847:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100462592, 100466688) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.360850:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24527 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.360864:0:18302:0:(rw26.c:858:ll_write_end()) pos 100462592, len 4096, copied 4096 00000080:00200000:0.0:1775764896.360866:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000dc05b1c9[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.360868:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b04e9e32 17ffffc0001001 3:0 ffff90c212833b60 24527 lru 00000080:00200000:0.0:1775764896.360961:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b0980b81 24527: 1< 0 - - > 2< 100462592 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 151552 - | + - + - > 5< + - + - | 0 - | 4047 - -> 00000080:00200000:0.0:1775764896.360964:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000dc05b1c9 00000080:00200000:0.0:1775764896.360965:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.360968:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.360977:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360982:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360987:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.360996:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.361004:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.361006:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.361007:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.361014:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.361016:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.361017:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.361020:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361025:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.361031:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.361032:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.361034:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.361036:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24527, 24527] 00000080:00200000:0.0:1775764896.361043:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.361045:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.361064:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100466688 i_blocks 90112 00000080:00200000:0.0:1775764896.361066:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.361077:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100466688, count: 4096 00000080:00200000:0.0:1775764896.361083:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100466688, bytes: 4096 00000080:00200000:0.0:1775764896.361085:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.361088:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24529, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.361089:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100466688, 100470784) 0 0 00000080:00200000:0.0:1775764896.361090:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.361094:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24528, 24528] 00020000:00200000:0.0:1775764896.361096:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100466688 chunk: [100466688, 100470784) 100470784, 4096 00020000:00200000:0.0:1775764896.361097:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.361104:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24528, 24528] 00000008:00010000:0.0:1775764896.361107:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.361109:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.361126:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361133:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100466688 100470783) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361138:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361143:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361147:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.361152:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100466688, 100470784) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.361155:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24528 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.361168:0:18302:0:(rw26.c:858:ll_write_end()) pos 100466688, len 4096, copied 4096 00000080:00200000:0.0:1775764896.361171:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f9b8614b[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.361172:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002b416b9c 17ffffc0001001 3:0 ffff90c212833000 24528 lru 00000080:00200000:0.0:1775764896.361177:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f6ef2f6d 24528: 1< 0 - - > 2< 100466688 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 147456 - | + - + - > 5< + - + - | 0 - | 4048 - -> 00000080:00200000:0.0:1775764896.361178:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f9b8614b 00000080:00200000:0.0:1775764896.361179:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.361180:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.361186:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.361191:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.361195:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.361199:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.361206:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.361207:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.361208:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.361214:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.361216:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.361217:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.361219:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361224:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.361232:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.361234:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.361236:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.361239:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24528, 24528] 00000080:00200000:0.0:1775764896.361248:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.361252:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.361280:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100470784 i_blocks 90112 00000080:00200000:0.0:1775764896.361284:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.361299:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100470784, count: 4096 00000080:00200000:0.0:1775764896.361310:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100470784, bytes: 4096 00000080:00200000:0.0:1775764896.361313:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.361317:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24530, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.361318:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100470784, 100474880) 0 0 00000080:00200000:0.0:1775764896.361321:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.361326:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24529, 24529] 00020000:00200000:0.0:1775764896.361328:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100470784 chunk: [100470784, 100474880) 100474880, 4096 00020000:00200000:0.0:1775764896.361331:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.361342:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24529, 24529] 00000008:00010000:0.0:1775764896.361347:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.361350:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.361358:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361367:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100470784 100474879) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361373:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361382:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361386:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.361391:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100470784, 100474880) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.361395:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24529 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.361408:0:18302:0:(rw26.c:858:ll_write_end()) pos 100470784, len 4096, copied 4096 00000080:00200000:0.0:1775764896.361410:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000530d62aa[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.361412:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000038501355 17ffffc0001001 3:0 ffff90c212833ee0 24529 lru 00000080:00200000:0.0:1775764896.361415:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000000cb2fbd 24529: 1< 0 - - > 2< 100470784 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 143360 - | + - + - > 5< + - + - | 0 - | 4049 - -> 00000080:00200000:0.0:1775764896.361416:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000530d62aa 00000080:00200000:0.0:1775764896.361417:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.361418:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.361424:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.361428:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.361432:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.361436:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.361447:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.361448:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.361450:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.361457:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.361459:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.361460:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.361463:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361468:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.361647:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.361649:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.361653:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.361655:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24529, 24529] 00000080:00200000:0.0:1775764896.361662:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.361665:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.361685:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100474880 i_blocks 90112 00000080:00200000:0.0:1775764896.361688:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.361699:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100474880, count: 4096 00000080:00200000:0.0:1775764896.361705:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100474880, bytes: 4096 00000080:00200000:0.0:1775764896.361707:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.361709:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24531, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.361710:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100474880, 100478976) 0 0 00000080:00200000:0.0:1775764896.361712:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.361715:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24530, 24530] 00020000:00200000:0.0:1775764896.361718:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100474880 chunk: [100474880, 100478976) 100478976, 4096 00020000:00200000:0.0:1775764896.361719:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.361727:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24530, 24530] 00000008:00010000:0.0:1775764896.361730:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.361732:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.361737:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361743:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100474880 100478975) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361748:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361753:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361757:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.361761:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100474880, 100478976) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.361764:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24530 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.361778:0:18302:0:(rw26.c:858:ll_write_end()) pos 100474880, len 4096, copied 4096 00000080:00200000:0.0:1775764896.361781:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000303145b4[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.361782:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000012691cb6 17ffffc0001001 3:0 ffff90c2128330e0 24530 lru 00000080:00200000:0.0:1775764896.361786:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000032dac588 24530: 1< 0 - - > 2< 100474880 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 139264 - | + - + - > 5< + - + - | 0 - | 4050 - -> 00000080:00200000:0.0:1775764896.361787:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000303145b4 00000080:00200000:0.0:1775764896.361788:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.361790:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.361795:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.361801:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.361805:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.361809:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.361816:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.361817:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.361818:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.361825:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.361827:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.361828:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.361831:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.361836:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.361841:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.361842:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.361844:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.361846:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24530, 24530] 00000080:00200000:0.0:1775764896.361854:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.361855:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.361959:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100478976 i_blocks 90112 00000080:00200000:0.0:1775764896.361963:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.361976:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100478976, count: 4096 00000080:00200000:0.0:1775764896.361984:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100478976, bytes: 4096 00000080:00200000:0.0:1775764896.361985:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.361988:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24532, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.361989:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100478976, 100483072) 0 0 00000080:00200000:0.0:1775764896.361991:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.361994:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24531, 24531] 00020000:00200000:0.0:1775764896.361997:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100478976 chunk: [100478976, 100483072) 100483072, 4096 00020000:00200000:0.0:1775764896.361998:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.362005:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24531, 24531] 00000008:00010000:0.0:1775764896.362008:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.362010:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.362015:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362021:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100478976 100483071) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362027:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362032:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362036:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.362041:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100478976, 100483072) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.362045:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24531 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.362057:0:18302:0:(rw26.c:858:ll_write_end()) pos 100478976, len 4096, copied 4096 00000080:00200000:0.0:1775764896.362060:0:18302:0:(rw26.c:916:ll_write_end()) page@000000000ed08e96[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.362062:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000088abe84 17ffffc0001001 3:0 ffff90c2128331c0 24531 lru 00000080:00200000:0.0:1775764896.362065:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000089bcb0f9 24531: 1< 0 - - > 2< 100478976 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 135168 - | + - + - > 5< + - + - | 0 - | 4051 - -> 00000080:00200000:0.0:1775764896.362067:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000000ed08e96 00000080:00200000:0.0:1775764896.362068:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.362069:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.362074:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362079:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362083:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362087:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.362094:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.362095:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.362097:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.362104:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.362105:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.362106:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.362109:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362126:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.362132:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.362133:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.362136:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.362138:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24531, 24531] 00000080:00200000:0.0:1775764896.362145:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.362147:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.362166:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100483072 i_blocks 90112 00000080:00200000:0.0:1775764896.362169:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.362179:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100483072, count: 4096 00000080:00200000:0.0:1775764896.362185:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100483072, bytes: 4096 00000080:00200000:0.0:1775764896.362187:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.362189:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24533, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.362190:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100483072, 100487168) 0 0 00000080:00200000:0.0:1775764896.362192:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.362196:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24532, 24532] 00020000:00200000:0.0:1775764896.362198:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100483072 chunk: [100483072, 100487168) 100487168, 4096 00020000:00200000:0.0:1775764896.362199:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.362206:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24532, 24532] 00000008:00010000:0.0:1775764896.362209:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.362210:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.362216:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362222:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100483072 100487167) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362227:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362232:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362236:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.362240:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100483072, 100487168) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.362244:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24532 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.362259:0:18302:0:(rw26.c:858:ll_write_end()) pos 100483072, len 4096, copied 4096 00000080:00200000:0.0:1775764896.362263:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ec416bde[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.362266:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000061afb8bb 17ffffc0001001 3:0 ffff90c2128337e0 24532 lru 00000080:00200000:0.0:1775764896.362271:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008c324821 24532: 1< 0 - - > 2< 100483072 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 131072 - | + - + - > 5< + - + - | 0 - | 4052 - -> 00000080:00200000:0.0:1775764896.362273:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ec416bde 00000080:00200000:0.0:1775764896.362274:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.362277:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.362285:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362293:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362300:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362312:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.362324:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.362327:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.362329:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.362339:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.362341:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.362344:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.362348:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362357:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.362365:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.362367:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.362372:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.362373:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24532, 24532] 00000080:00200000:0.0:1775764896.362381:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.362383:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.362402:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100487168 i_blocks 90112 00000080:00200000:0.0:1775764896.362404:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.362415:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100487168, count: 4096 00000080:00200000:0.0:1775764896.362421:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100487168, bytes: 4096 00000080:00200000:0.0:1775764896.362423:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.362426:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24534, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.362427:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100487168, 100491264) 0 0 00000080:00200000:0.0:1775764896.362429:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.362432:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24533, 24533] 00020000:00200000:0.0:1775764896.362434:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100487168 chunk: [100487168, 100491264) 100491264, 4096 00020000:00200000:0.0:1775764896.362436:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.362442:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24533, 24533] 00000008:00010000:0.0:1775764896.362445:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.362447:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.362453:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362458:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100487168 100491263) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362464:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362468:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362537:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.362543:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100487168, 100491264) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.362547:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24533 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.362572:0:18302:0:(rw26.c:858:ll_write_end()) pos 100487168, len 4096, copied 4096 00000080:00200000:0.0:1775764896.362575:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f094717f[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.362577:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000003dae94c 17ffffc0001001 3:0 ffff90c212837c40 24533 lru 00000080:00200000:0.0:1775764896.362581:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000009de05e31 24533: 1< 0 - - > 2< 100487168 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 126976 - | + - + - > 5< + - + - | 0 - | 4053 - -> 00000080:00200000:0.0:1775764896.362582:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f094717f 00000080:00200000:0.0:1775764896.362583:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.362585:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.362590:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362596:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362600:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362604:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.362611:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.362612:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.362614:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.362620:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.362622:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.362623:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.362625:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362630:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.362639:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.362640:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.362643:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.362645:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24533, 24533] 00000080:00200000:0.0:1775764896.362651:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.362653:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.362672:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100491264 i_blocks 90112 00000080:00200000:0.0:1775764896.362674:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.362685:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100491264, count: 4096 00000080:00200000:0.0:1775764896.362692:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100491264, bytes: 4096 00000080:00200000:0.0:1775764896.362693:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.362696:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24535, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.362697:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100491264, 100495360) 0 0 00000080:00200000:0.0:1775764896.362698:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.362702:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24534, 24534] 00020000:00200000:0.0:1775764896.362704:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100491264 chunk: [100491264, 100495360) 100495360, 4096 00020000:00200000:0.0:1775764896.362705:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.362712:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24534, 24534] 00000008:00010000:0.0:1775764896.362715:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.362716:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.362721:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362727:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100491264 100495359) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362733:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362741:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362745:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.362749:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100491264, 100495360) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.362752:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24534 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.362770:0:18302:0:(rw26.c:858:ll_write_end()) pos 100491264, len 4096, copied 4096 00000080:00200000:0.0:1775764896.362772:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000006df1c19[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.362774:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004904f414 17ffffc0001021 2:0 ffff90c2128378c0 24534 lru 00000080:00200000:0.0:1775764896.362777:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000345bfc84 24534: 1< 0 - - > 2< 100491264 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 122880 - | + - + - > 5< + - + - | 0 - | 4054 - -> 00000080:00200000:0.0:1775764896.362778:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000006df1c19 00000080:00200000:0.0:1775764896.362779:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.362780:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.362786:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362791:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362795:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.362799:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.362809:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.362810:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.362812:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.362819:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.362821:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.362822:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.362825:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.362830:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.362835:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.362836:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.362839:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.362840:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24534, 24534] 00000080:00200000:0.0:1775764896.362847:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.362849:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.362867:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100495360 i_blocks 90112 00000080:00200000:0.0:1775764896.362898:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.362977:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100495360, count: 4096 00000080:00200000:0.0:1775764896.362985:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100495360, bytes: 4096 00000080:00200000:0.0:1775764896.362987:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.362990:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24536, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.362991:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100495360, 100499456) 0 0 00000080:00200000:0.0:1775764896.362992:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.362996:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24535, 24535] 00020000:00200000:0.0:1775764896.362999:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100495360 chunk: [100495360, 100499456) 100499456, 4096 00020000:00200000:0.0:1775764896.363000:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.363007:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24535, 24535] 00000008:00010000:0.0:1775764896.363011:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.363013:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.363018:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363024:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100495360 100499455) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363030:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363034:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363038:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.363043:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100495360, 100499456) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.363046:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24535 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.363059:0:18302:0:(rw26.c:858:ll_write_end()) pos 100495360, len 4096, copied 4096 00000080:00200000:0.0:1775764896.363061:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000bc0d63c3[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.363063:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000004bbc5fec 17ffffc0001001 3:0 ffff90c212837b60 24535 lru 00000080:00200000:0.0:1775764896.363067:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003eaee9f1 24535: 1< 0 - - > 2< 100495360 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 118784 - | + - + - > 5< + - + - | 0 - | 4055 - -> 00000080:00200000:0.0:1775764896.363069:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000bc0d63c3 00000080:00200000:0.0:1775764896.363070:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.363071:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.363077:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363082:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363086:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363090:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.363097:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.363098:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.363099:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.363106:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.363108:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.363109:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.363121:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363126:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.363133:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.363134:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.363137:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.363139:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24535, 24535] 00000080:00200000:0.0:1775764896.363147:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.363148:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.363167:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100499456 i_blocks 90112 00000080:00200000:0.0:1775764896.363170:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.363181:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100499456, count: 4096 00000080:00200000:0.0:1775764896.363187:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100499456, bytes: 4096 00000080:00200000:0.0:1775764896.363188:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.363191:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24537, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.363192:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100499456, 100503552) 0 0 00000080:00200000:0.0:1775764896.363193:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.363197:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24536, 24536] 00020000:00200000:0.0:1775764896.363199:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100499456 chunk: [100499456, 100503552) 100503552, 4096 00020000:00200000:0.0:1775764896.363200:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.363207:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24536, 24536] 00000008:00010000:0.0:1775764896.363210:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.363211:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.363216:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363222:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100499456 100503551) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363228:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363232:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363236:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.363242:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100499456, 100503552) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.363246:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24536 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.363263:0:18302:0:(rw26.c:858:ll_write_end()) pos 100499456, len 4096, copied 4096 00000080:00200000:0.0:1775764896.363267:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000a2991fc3[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.363270:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000d3e58e22 17ffffc0001001 3:0 ffff90c212837000 24536 lru 00000080:00200000:0.0:1775764896.363275:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000044356ab6 24536: 1< 0 - - > 2< 100499456 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 114688 - | + - + - > 5< + - + - | 0 - | 4056 - -> 00000080:00200000:0.0:1775764896.363277:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000a2991fc3 00000080:00200000:0.0:1775764896.363278:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.363280:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.363288:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363296:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363303:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363308:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.363316:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.363317:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.363319:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.363325:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.363327:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.363329:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.363332:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363337:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.363342:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.363343:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.363346:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.363348:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24536, 24536] 00000080:00200000:0.0:1775764896.363355:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.363357:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.363375:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100503552 i_blocks 90112 00000080:00200000:0.0:1775764896.363377:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.363388:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100503552, count: 4096 00000080:00200000:0.0:1775764896.363394:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100503552, bytes: 4096 00000080:00200000:0.0:1775764896.363396:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.363398:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24538, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.363399:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100503552, 100507648) 0 0 00000080:00200000:0.0:1775764896.363401:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.363404:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24537, 24537] 00020000:00200000:0.0:1775764896.363406:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100503552 chunk: [100503552, 100507648) 100507648, 4096 00020000:00200000:0.0:1775764896.363407:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.363414:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24537, 24537] 00000008:00010000:0.0:1775764896.363417:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.363419:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.363424:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363430:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100503552 100507647) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363436:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363440:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363444:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.363449:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100503552, 100507648) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.363452:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24537 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.363464:0:18302:0:(rw26.c:858:ll_write_end()) pos 100503552, len 4096, copied 4096 00000080:00200000:0.0:1775764896.363467:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000036286633[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.363469:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000700ec8ae 17ffffc0001001 3:0 ffff90c212837ee0 24537 lru 00000080:00200000:0.0:1775764896.363537:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000b0ed9a00 24537: 1< 0 - - > 2< 100503552 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 110592 - | + - + - > 5< + - + - | 0 - | 4057 - -> 00000080:00200000:0.0:1775764896.363538:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000036286633 00000080:00200000:0.0:1775764896.363539:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.363541:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.363547:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363552:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363556:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363564:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.363572:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.363574:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.363575:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.363582:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.363583:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.363585:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.363587:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363593:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.363598:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.363599:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.363601:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.363603:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24537, 24537] 00000080:00200000:0.0:1775764896.363610:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.363612:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.363630:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100507648 i_blocks 90112 00000080:00200000:0.0:1775764896.363633:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.363643:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100507648, count: 4096 00000080:00200000:0.0:1775764896.363649:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100507648, bytes: 4096 00000080:00200000:0.0:1775764896.363651:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.363654:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24539, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.363655:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100507648, 100511744) 0 0 00000080:00200000:0.0:1775764896.363656:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.363660:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24538, 24538] 00020000:00200000:0.0:1775764896.363662:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100507648 chunk: [100507648, 100511744) 100511744, 4096 00020000:00200000:0.0:1775764896.363663:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.363670:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24538, 24538] 00000008:00010000:0.0:1775764896.363673:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.363675:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.363680:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363685:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100507648 100511743) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363691:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363696:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363699:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.363703:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100507648, 100511744) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.363707:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24538 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.363718:0:18302:0:(rw26.c:858:ll_write_end()) pos 100507648, len 4096, copied 4096 00000080:00200000:0.0:1775764896.363721:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000cc496e87[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.363723:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000b339e561 17ffffc0001001 3:0 ffff90c2128370e0 24538 lru 00000080:00200000:0.0:1775764896.363727:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000075d821e2 24538: 1< 0 - - > 2< 100507648 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 106496 - | + - + - > 5< + - + - | 0 - | 4058 - -> 00000080:00200000:0.0:1775764896.363728:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000cc496e87 00000080:00200000:0.0:1775764896.363729:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.363730:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.363735:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363740:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363744:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.363748:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.363754:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.363756:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.363757:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.363763:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.363765:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.363766:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.363768:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363773:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.363781:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.363783:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.363786:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.363787:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24538, 24538] 00000080:00200000:0.0:1775764896.363794:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.363796:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.363814:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100511744 i_blocks 90112 00000080:00200000:0.0:1775764896.363817:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.363827:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100511744, count: 4096 00000080:00200000:0.0:1775764896.363833:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100511744, bytes: 4096 00000080:00200000:0.0:1775764896.363835:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.363837:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24540, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.363838:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100511744, 100515840) 0 0 00000080:00200000:0.0:1775764896.363840:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.363843:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24539, 24539] 00020000:00200000:0.0:1775764896.363845:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100511744 chunk: [100511744, 100515840) 100515840, 4096 00020000:00200000:0.0:1775764896.363847:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.363853:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24539, 24539] 00000008:00010000:0.0:1775764896.363856:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.363858:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.363863:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363868:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100511744 100515839) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363892:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363962:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.363967:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.363972:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100511744, 100515840) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.363976:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24539 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.363989:0:18302:0:(rw26.c:858:ll_write_end()) pos 100511744, len 4096, copied 4096 00000080:00200000:0.0:1775764896.363992:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000f768fd72[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.363994:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000002ed705bb 17ffffc0001001 3:0 ffff90c2128371c0 24539 lru 00000080:00200000:0.0:1775764896.363997:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000811f4902 24539: 1< 0 - - > 2< 100511744 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 102400 - | + - + - > 5< + - + - | 0 - | 4059 - -> 00000080:00200000:0.0:1775764896.363999:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000f768fd72 00000080:00200000:0.0:1775764896.364000:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.364001:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.364006:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364011:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364015:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364019:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.364030:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.364032:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.364033:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.364040:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.364042:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.364043:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.364046:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364051:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.364056:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.364058:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.364060:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.364062:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24539, 24539] 00000080:00200000:0.0:1775764896.364069:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.364070:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.364089:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100515840 i_blocks 90112 00000080:00200000:0.0:1775764896.364091:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.364102:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100515840, count: 4096 00000080:00200000:0.0:1775764896.364109:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100515840, bytes: 4096 00000080:00200000:0.0:1775764896.364110:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.364124:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24541, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.364125:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100515840, 100519936) 0 0 00000080:00200000:0.0:1775764896.364127:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.364130:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24540, 24540] 00020000:00200000:0.0:1775764896.364133:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100515840 chunk: [100515840, 100519936) 100519936, 4096 00020000:00200000:0.0:1775764896.364134:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.364142:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24540, 24540] 00000008:00010000:0.0:1775764896.364145:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.364147:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.364152:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364158:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100515840 100519935) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364164:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364168:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364172:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.364176:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100515840, 100519936) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.364179:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24540 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.364191:0:18302:0:(rw26.c:858:ll_write_end()) pos 100515840, len 4096, copied 4096 00000080:00200000:0.0:1775764896.364193:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000026929107[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.364195:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000552f9f10 17ffffc0001001 3:0 ffff90c2128377e0 24540 lru 00000080:00200000:0.0:1775764896.364198:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000008985539b 24540: 1< 0 - - > 2< 100515840 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 98304 - | + - + - > 5< + - + - | 0 - | 4060 - -> 00000080:00200000:0.0:1775764896.364199:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000026929107 00000080:00200000:0.0:1775764896.364200:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.364201:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.364207:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364212:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364216:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364220:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.364227:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.364228:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.364230:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.364236:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.364237:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.364239:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.364241:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364248:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.364256:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.364259:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.364263:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.364265:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24540, 24540] 00000080:00200000:0.0:1775764896.364276:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.364279:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.364303:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100519936 i_blocks 90112 00000080:00200000:0.0:1775764896.364306:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.364317:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100519936, count: 4096 00000080:00200000:0.0:1775764896.364323:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100519936, bytes: 4096 00000080:00200000:0.0:1775764896.364325:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.364327:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24542, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.364329:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100519936, 100524032) 0 0 00000080:00200000:0.0:1775764896.364330:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.364334:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24541, 24541] 00020000:00200000:0.0:1775764896.364336:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100519936 chunk: [100519936, 100524032) 100524032, 4096 00020000:00200000:0.0:1775764896.364337:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.364344:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24541, 24541] 00000008:00010000:0.0:1775764896.364347:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.364349:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.364354:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364360:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100519936 100524031) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364365:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364370:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364374:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.364379:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100519936, 100524032) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.364382:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24541 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.364394:0:18302:0:(rw26.c:858:ll_write_end()) pos 100519936, len 4096, copied 4096 00000080:00200000:0.0:1775764896.364397:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001d0b9933[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.364398:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000035b58036 17ffffc0001001 3:0 ffff90c2128379a0 24541 lru 00000080:00200000:0.0:1775764896.364402:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001f30f09b 24541: 1< 0 - - > 2< 100519936 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 94208 - | + - + - > 5< + - + - | 0 - | 4061 - -> 00000080:00200000:0.0:1775764896.364403:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001d0b9933 00000080:00200000:0.0:1775764896.364404:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.364406:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.364410:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364415:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364420:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364424:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.364430:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.364432:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.364433:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.364439:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.364440:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.364442:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.364445:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364449:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.364454:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.364456:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.364459:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.364460:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24541, 24541] 00000080:00200000:0.0:1775764896.364467:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.364469:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.364549:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100524032 i_blocks 90112 00000080:00200000:0.0:1775764896.364552:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.364563:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100524032, count: 4096 00000080:00200000:0.0:1775764896.364569:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100524032, bytes: 4096 00000080:00200000:0.0:1775764896.364571:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.364573:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24543, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.364574:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100524032, 100528128) 0 0 00000080:00200000:0.0:1775764896.364576:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.364580:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24542, 24542] 00020000:00200000:0.0:1775764896.364582:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100524032 chunk: [100524032, 100528128) 100528128, 4096 00020000:00200000:0.0:1775764896.364583:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.364590:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24542, 24542] 00000008:00010000:0.0:1775764896.364593:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.364595:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.364600:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364606:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100524032 100528127) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364612:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364617:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364620:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.364625:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100524032, 100528128) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.364628:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24542 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.364641:0:18302:0:(rw26.c:858:ll_write_end()) pos 100524032, len 4096, copied 4096 00000080:00200000:0.0:1775764896.364644:0:18302:0:(rw26.c:916:ll_write_end()) page@000000007886e573[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.364646:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000020fc9174 17ffffc0001001 3:0 ffff90c2128372a0 24542 lru 00000080:00200000:0.0:1775764896.364650:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000056d3c363 24542: 1< 0 - - > 2< 100524032 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 90112 - | + - + - > 5< + - + - | 0 - | 4062 - -> 00000080:00200000:0.0:1775764896.364651:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000007886e573 00000080:00200000:0.0:1775764896.364652:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.364653:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.364658:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364663:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364667:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364675:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.364682:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.364683:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.364684:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.364691:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.364692:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.364694:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.364696:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364701:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.364706:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.364708:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.364710:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.364712:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24542, 24542] 00000080:00200000:0.0:1775764896.364719:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.364720:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.364739:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100528128 i_blocks 90112 00000080:00200000:0.0:1775764896.364741:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.364751:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100528128, count: 4096 00000080:00200000:0.0:1775764896.364757:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100528128, bytes: 4096 00000080:00200000:0.0:1775764896.364759:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.364762:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24544, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.364763:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100528128, 100532224) 0 0 00000080:00200000:0.0:1775764896.364764:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.364768:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24543, 24543] 00020000:00200000:0.0:1775764896.364770:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100528128 chunk: [100528128, 100532224) 100532224, 4096 00020000:00200000:0.0:1775764896.364771:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.364777:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24543, 24543] 00000008:00010000:0.0:1775764896.364780:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.364782:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.364787:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364793:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100528128 100532223) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364798:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364803:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364806:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.364810:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100528128, 100532224) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.364813:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24543 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.364825:0:18302:0:(rw26.c:858:ll_write_end()) pos 100528128, len 4096, copied 4096 00000080:00200000:0.0:1775764896.364827:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000ee89d306[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.364829:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000df98dccb 17ffffc0001001 3:0 ffff90c212837d20 24543 lru 00000080:00200000:0.0:1775764896.364833:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000004b955f26 24543: 1< 0 - - > 2< 100528128 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 86016 - | + - + - > 5< + - + - | 0 - | 4063 - -> 00000080:00200000:0.0:1775764896.364834:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000ee89d306 00000080:00200000:0.0:1775764896.364835:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.364836:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.364841:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364846:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364850:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.364854:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.364861:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.364862:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.364864:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.364894:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.364897:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.364969:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.364973:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.364978:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.364986:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.364988:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.364991:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.364993:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24543, 24543] 00000080:00200000:0.0:1775764896.365001:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.365003:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.365022:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100532224 i_blocks 90112 00000080:00200000:0.0:1775764896.365025:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.365036:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100532224, count: 4096 00000080:00200000:0.0:1775764896.365042:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100532224, bytes: 4096 00000080:00200000:0.0:1775764896.365044:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.365046:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24545, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.365047:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100532224, 100536320) 0 0 00000080:00200000:0.0:1775764896.365049:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.365052:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24544, 24544] 00020000:00200000:0.0:1775764896.365054:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100532224 chunk: [100532224, 100536320) 100536320, 4096 00020000:00200000:0.0:1775764896.365055:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.365062:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24544, 24544] 00000008:00010000:0.0:1775764896.365065:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.365067:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.365072:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365077:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100532224 100536319) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365083:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365091:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365095:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.365100:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100532224, 100536320) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.365103:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24544 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.365126:0:18302:0:(rw26.c:858:ll_write_end()) pos 100532224, len 4096, copied 4096 00000080:00200000:0.0:1775764896.365130:0:18302:0:(rw26.c:916:ll_write_end()) page@000000008bffda54[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.365131:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006041ca6c 17ffffc0001001 3:0 ffff90c212837380 24544 lru 00000080:00200000:0.0:1775764896.365135:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000a11c4342 24544: 1< 0 - - > 2< 100532224 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 81920 - | + - + - > 5< + - + - | 0 - | 4064 - -> 00000080:00200000:0.0:1775764896.365136:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000008bffda54 00000080:00200000:0.0:1775764896.365136:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.365138:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.365144:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365149:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365153:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365157:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.365167:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.365169:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.365171:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.365178:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.365179:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.365181:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.365184:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365189:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.365194:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.365195:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.365198:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.365200:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24544, 24544] 00000080:00200000:0.0:1775764896.365207:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.365208:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.365227:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100536320 i_blocks 90112 00000080:00200000:0.0:1775764896.365229:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.365240:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100536320, count: 4096 00000080:00200000:0.0:1775764896.365248:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100536320, bytes: 4096 00000080:00200000:0.0:1775764896.365251:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.365255:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24546, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.365256:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100536320, 100540416) 0 0 00000080:00200000:0.0:1775764896.365259:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.365264:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24545, 24545] 00020000:00200000:0.0:1775764896.365268:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100536320 chunk: [100536320, 100540416) 100540416, 4096 00020000:00200000:0.0:1775764896.365270:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.365281:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24545, 24545] 00000008:00010000:0.0:1775764896.365286:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.365288:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.365295:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365303:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100536320 100540415) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365311:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365318:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365324:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.365332:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100536320, 100540416) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.365337:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24545 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.365350:0:18302:0:(rw26.c:858:ll_write_end()) pos 100536320, len 4096, copied 4096 00000080:00200000:0.0:1775764896.365353:0:18302:0:(rw26.c:916:ll_write_end()) page@000000002952ba7c[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.365355:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000baa7fd0f 17ffffc0001001 3:0 ffff90c212837620 24545 lru 00000080:00200000:0.0:1775764896.365359:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000cc8ba33 24545: 1< 0 - - > 2< 100536320 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 77824 - | + - + - > 5< + - + - | 0 - | 4065 - -> 00000080:00200000:0.0:1775764896.365360:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000002952ba7c 00000080:00200000:0.0:1775764896.365361:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.365362:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.365368:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365373:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365377:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365381:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.365388:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.365389:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.365391:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.365397:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.365398:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.365400:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.365402:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365406:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.365412:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.365413:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.365416:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.365418:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24545, 24545] 00000080:00200000:0.0:1775764896.365425:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.365427:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.365445:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100540416 i_blocks 90112 00000080:00200000:0.0:1775764896.365448:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.365461:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100540416, count: 4096 00000080:00200000:0.0:1775764896.365550:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100540416, bytes: 4096 00000080:00200000:0.0:1775764896.365554:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.365557:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24547, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.365559:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100540416, 100544512) 0 0 00000080:00200000:0.0:1775764896.365561:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.365567:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24546, 24546] 00020000:00200000:0.0:1775764896.365570:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100540416 chunk: [100540416, 100544512) 100544512, 4096 00020000:00200000:0.0:1775764896.365572:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.365584:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24546, 24546] 00000008:00010000:0.0:1775764896.365589:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.365592:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.365599:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365608:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100540416 100544511) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365616:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365622:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365628:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.365636:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100540416, 100544512) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.365641:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24546 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.365657:0:18302:0:(rw26.c:858:ll_write_end()) pos 100540416, len 4096, copied 4096 00000080:00200000:0.0:1775764896.365661:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c71b1384[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.365664:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000c8c06505 17ffffc0001001 3:0 ffff90c212837e00 24546 lru 00000080:00200000:0.0:1775764896.365670:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f6e3a907 24546: 1< 0 - - > 2< 100540416 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 73728 - | + - + - > 5< + - + - | 0 - | 4066 - -> 00000080:00200000:0.0:1775764896.365672:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c71b1384 00000080:00200000:0.0:1775764896.365674:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.365676:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.365683:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365688:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365692:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365696:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.365704:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.365705:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.365707:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.365714:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.365715:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.365717:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.365720:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365725:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.365731:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.365732:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.365734:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.365736:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24546, 24546] 00000080:00200000:0.0:1775764896.365743:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.365745:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.365764:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100544512 i_blocks 90112 00000080:00200000:0.0:1775764896.365766:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.365777:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100544512, count: 4096 00000080:00200000:0.0:1775764896.365783:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100544512, bytes: 4096 00000080:00200000:0.0:1775764896.365785:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.365787:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24548, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.365788:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100544512, 100548608) 0 0 00000080:00200000:0.0:1775764896.365790:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.365793:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24547, 24547] 00020000:00200000:0.0:1775764896.365795:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100544512 chunk: [100544512, 100548608) 100548608, 4096 00020000:00200000:0.0:1775764896.365797:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.365803:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24547, 24547] 00000008:00010000:0.0:1775764896.365806:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.365808:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.365813:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365819:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100544512 100548607) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365825:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365829:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.365833:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.365837:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100544512, 100548608) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.365840:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24547 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.365852:0:18302:0:(rw26.c:858:ll_write_end()) pos 100544512, len 4096, copied 4096 00000080:00200000:0.0:1775764896.365855:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000046d932e4[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.365856:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000006aac0c2b 17ffffc0001001 3:0 ffff90c212837540 24547 lru 00000080:00200000:0.0:1775764896.365859:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002b52419e 24547: 1< 0 - - > 2< 100544512 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 69632 - | + - + - > 5< + - + - | 0 - | 4067 - -> 00000080:00200000:0.0:1775764896.365860:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000046d932e4 00000080:00200000:0.0:1775764896.365861:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.365862:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.365867:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365898:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365971:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.365981:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.365990:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.365992:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.365993:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.366001:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.366004:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.366007:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.366010:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366019:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.366027:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.366029:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.366033:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.366036:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24547, 24547] 00000080:00200000:0.0:1775764896.366047:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.366050:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.366079:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100548608 i_blocks 90112 00000080:00200000:0.0:1775764896.366222:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.366236:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100548608, count: 4096 00000080:00200000:0.0:1775764896.366245:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100548608, bytes: 4096 00000080:00200000:0.0:1775764896.366248:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.366252:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24549, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.366254:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100548608, 100552704) 0 0 00000080:00200000:0.0:1775764896.366256:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.366262:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24548, 24548] 00020000:00200000:0.0:1775764896.366265:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100548608 chunk: [100548608, 100552704) 100552704, 4096 00020000:00200000:0.0:1775764896.366267:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.366279:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24548, 24548] 00000008:00010000:0.0:1775764896.366285:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.366287:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.366295:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366304:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100548608 100552703) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366312:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366320:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366327:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.366336:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100548608, 100552704) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.366341:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24548 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.366356:0:18302:0:(rw26.c:858:ll_write_end()) pos 100548608, len 4096, copied 4096 00000080:00200000:0.0:1775764896.366359:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000b30773d0[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.366361:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000a3a644f6 17ffffc0001001 3:0 ffff90c212837460 24548 lru 00000080:00200000:0.0:1775764896.366365:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000546c36da 24548: 1< 0 - - > 2< 100548608 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 65536 - | + - + - > 5< + - + - | 0 - | 4068 - -> 00000080:00200000:0.0:1775764896.366366:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000b30773d0 00000080:00200000:0.0:1775764896.366367:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.366368:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.366374:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.366379:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.366383:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.366387:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.366394:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.366396:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.366397:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.366404:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.366406:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.366407:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.366410:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366415:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.366424:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.366425:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.366428:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.366430:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24548, 24548] 00000080:00200000:0.0:1775764896.366437:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.366439:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.366458:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100552704 i_blocks 90112 00000080:00200000:0.0:1775764896.366461:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.366531:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100552704, count: 4096 00000080:00200000:0.0:1775764896.366543:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100552704, bytes: 4096 00000080:00200000:0.0:1775764896.366546:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.366550:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24550, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.366551:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100552704, 100556800) 0 0 00000080:00200000:0.0:1775764896.366554:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.366559:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24549, 24549] 00020000:00200000:0.0:1775764896.366562:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100552704 chunk: [100552704, 100556800) 100556800, 4096 00020000:00200000:0.0:1775764896.366564:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.366576:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24549, 24549] 00000008:00010000:0.0:1775764896.366582:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.366584:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.366592:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366601:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100552704 100556799) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366610:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366618:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366622:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.366628:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100552704, 100556800) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.366631:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24549 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.366649:0:18302:0:(rw26.c:858:ll_write_end()) pos 100552704, len 4096, copied 4096 00000080:00200000:0.0:1775764896.366652:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000df05ac26[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.366654:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000000f451833 17ffffc0001021 2:0 ffff90c212837a80 24549 lru 00000080:00200000:0.0:1775764896.366657:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@0000000001e7a96b 24549: 1< 0 - - > 2< 100552704 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 61440 - | + - + - > 5< + - + - | 0 - | 4069 - -> 00000080:00200000:0.0:1775764896.366658:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000df05ac26 00000080:00200000:0.0:1775764896.366659:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.366661:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.366666:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.366671:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.366675:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.366679:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.366689:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.366691:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.366693:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.366700:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.366701:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.366703:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.366706:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366711:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.366716:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.366717:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.366720:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.366721:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24549, 24549] 00000080:00200000:0.0:1775764896.366728:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.366730:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.366749:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100556800 i_blocks 90112 00000080:00200000:0.0:1775764896.366751:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.366762:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100556800, count: 4096 00000080:00200000:0.0:1775764896.366768:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100556800, bytes: 4096 00000080:00200000:0.0:1775764896.366770:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.366772:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24551, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.366774:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100556800, 100560896) 0 0 00000080:00200000:0.0:1775764896.366775:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.366778:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24550, 24550] 00020000:00200000:0.0:1775764896.366781:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100556800 chunk: [100556800, 100560896) 100560896, 4096 00020000:00200000:0.0:1775764896.366782:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.366788:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24550, 24550] 00000008:00010000:0.0:1775764896.366792:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.366793:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.366798:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366804:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100556800 100560895) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366809:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366814:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366817:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.366822:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100556800, 100560896) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.366824:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24550 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.366836:0:18302:0:(rw26.c:858:ll_write_end()) pos 100556800, len 4096, copied 4096 00000080:00200000:0.0:1775764896.366839:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000e00051e9[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.366841:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000049dac59e 17ffffc0001001 3:0 ffff90c212837700 24550 lru 00000080:00200000:0.0:1775764896.366844:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000001ed12839 24550: 1< 0 - - > 2< 100556800 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 57344 - | + - + - > 5< + - + - | 0 - | 4070 - -> 00000080:00200000:0.0:1775764896.366845:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000e00051e9 00000080:00200000:0.0:1775764896.366846:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.366847:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.366852:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.366857:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.366861:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.366865:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.366896:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.366966:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.366969:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.366977:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.366979:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.366980:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.366983:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.366988:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.366992:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.366994:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.366996:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.366998:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24550, 24550] 00000080:00200000:0.0:1775764896.367005:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.367007:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.367027:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100560896 i_blocks 90112 00000080:00200000:0.0:1775764896.367029:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.367040:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100560896, count: 4096 00000080:00200000:0.0:1775764896.367046:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100560896, bytes: 4096 00000080:00200000:0.0:1775764896.367048:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.367050:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24552, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.367051:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100560896, 100564992) 0 0 00000080:00200000:0.0:1775764896.367053:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.367056:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24551, 24551] 00020000:00200000:0.0:1775764896.367058:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100560896 chunk: [100560896, 100564992) 100564992, 4096 00020000:00200000:0.0:1775764896.367060:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.367066:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24551, 24551] 00000008:00010000:0.0:1775764896.367070:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.367071:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.367076:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367082:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100560896 100564991) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367087:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367092:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367095:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.367101:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100560896, 100564992) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.367104:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24551 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.367138:0:18302:0:(rw26.c:858:ll_write_end()) pos 100560896, len 4096, copied 4096 00000080:00200000:0.0:1775764896.367141:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c54ac700[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.367143:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000009cba4ce1 17ffffc0001001 3:0 ffff90c212838620 24551 lru 00000080:00200000:0.0:1775764896.367147:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000003deb66e9 24551: 1< 0 - - > 2< 100560896 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 53248 - | + - + - > 5< + - + - | 0 - | 4071 - -> 00000080:00200000:0.0:1775764896.367148:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c54ac700 00000080:00200000:0.0:1775764896.367149:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.367150:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.367156:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.367161:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.367165:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.367169:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.367176:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.367178:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.367179:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.367185:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.367186:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.367188:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.367191:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367196:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.367201:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.367202:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.367205:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.367207:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24551, 24551] 00000080:00200000:0.0:1775764896.367214:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.367216:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.367234:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100564992 i_blocks 90112 00000080:00200000:0.0:1775764896.367236:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.367250:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100564992, count: 4096 00000080:00200000:0.0:1775764896.367260:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100564992, bytes: 4096 00000080:00200000:0.0:1775764896.367263:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.367266:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24553, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.367268:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100564992, 100569088) 0 0 00000080:00200000:0.0:1775764896.367270:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.367276:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24552, 24552] 00020000:00200000:0.0:1775764896.367279:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100564992 chunk: [100564992, 100569088) 100569088, 4096 00020000:00200000:0.0:1775764896.367281:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.367292:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24552, 24552] 00000008:00010000:0.0:1775764896.367297:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.367299:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.367304:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367310:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100564992 100569087) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367316:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367321:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367324:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.367329:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100564992, 100569088) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.367333:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24552 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.367344:0:18302:0:(rw26.c:858:ll_write_end()) pos 100564992, len 4096, copied 4096 00000080:00200000:0.0:1775764896.367347:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c300472c[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.367349:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000529cfc2d 17ffffc0001001 3:0 ffff90c212838e00 24552 lru 00000080:00200000:0.0:1775764896.367353:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000000e1178c2 24552: 1< 0 - - > 2< 100564992 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 49152 - | + - + - > 5< + - + - | 0 - | 4072 - -> 00000080:00200000:0.0:1775764896.367354:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c300472c 00000080:00200000:0.0:1775764896.367355:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.367356:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.367361:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.367366:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.367370:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.367378:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.367385:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.367387:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.367388:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.367394:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.367396:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.367398:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.367400:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367405:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.367410:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.367412:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.367414:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.367416:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24552, 24552] 00000080:00200000:0.0:1775764896.367422:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.367424:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.367443:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100569088 i_blocks 90112 00000080:00200000:0.0:1775764896.367445:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.367455:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100569088, count: 4096 00000080:00200000:0.0:1775764896.367461:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100569088, bytes: 4096 00000080:00200000:0.0:1775764896.367463:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.367465:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24554, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.367466:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100569088, 100573184) 0 0 00000080:00200000:0.0:1775764896.367696:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.367701:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24553, 24553] 00020000:00200000:0.0:1775764896.367704:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100569088 chunk: [100569088, 100573184) 100573184, 4096 00020000:00200000:0.0:1775764896.367705:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.367715:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24553, 24553] 00000008:00010000:0.0:1775764896.367719:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.367721:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.367727:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367733:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100569088 100573183) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367739:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367743:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367747:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.367752:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100569088, 100573184) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.367755:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24553 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.367768:0:18302:0:(rw26.c:858:ll_write_end()) pos 100569088, len 4096, copied 4096 00000080:00200000:0.0:1775764896.367770:0:18302:0:(rw26.c:916:ll_write_end()) page@000000003f62e4ac[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.367772:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000627c97c2 17ffffc0001001 3:0 ffff90c212838540 24553 lru 00000080:00200000:0.0:1775764896.367776:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000007afc028e 24553: 1< 0 - - > 2< 100569088 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 45056 - | + - + - > 5< + - + - | 0 - | 4073 - -> 00000080:00200000:0.0:1775764896.367778:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000003f62e4ac 00000080:00200000:0.0:1775764896.367778:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.367780:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.367785:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.367790:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.367794:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.367798:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.367805:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.367807:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.367808:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.367816:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.367817:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.367819:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.367821:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.367825:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.367834:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.367835:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.367838:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.367840:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24553, 24553] 00000080:00200000:0.0:1775764896.367847:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.367849:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.367884:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100573184 i_blocks 90112 00000080:00200000:0.0:1775764896.367887:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.367959:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100573184, count: 4096 00000080:00200000:0.0:1775764896.367969:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100573184, bytes: 4096 00000080:00200000:0.0:1775764896.367971:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.367973:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24555, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.367974:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100573184, 100577280) 0 0 00000080:00200000:0.0:1775764896.367976:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.367980:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24554, 24554] 00020000:00200000:0.0:1775764896.367982:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100573184 chunk: [100573184, 100577280) 100577280, 4096 00020000:00200000:0.0:1775764896.367983:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.367990:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24554, 24554] 00000008:00010000:0.0:1775764896.367993:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.367995:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.368000:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368006:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100573184 100577279) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368011:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368020:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368024:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.368028:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100573184, 100577280) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.368031:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24554 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.368044:0:18302:0:(rw26.c:858:ll_write_end()) pos 100573184, len 4096, copied 4096 00000080:00200000:0.0:1775764896.368047:0:18302:0:(rw26.c:916:ll_write_end()) page@000000009f276f00[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.368048:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000001897c426 17ffffc0001001 3:0 ffff90c212838460 24554 lru 00000080:00200000:0.0:1775764896.368052:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000f3c0db5f 24554: 1< 0 - - > 2< 100573184 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 40960 - | + - + - > 5< + - + - | 0 - | 4074 - -> 00000080:00200000:0.0:1775764896.368053:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000009f276f00 00000080:00200000:0.0:1775764896.368053:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.368055:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.368060:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368065:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368069:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368073:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.368083:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.368085:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.368086:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.368093:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.368095:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.368096:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.368099:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368104:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.368109:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.368121:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.368125:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.368127:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24554, 24554] 00000080:00200000:0.0:1775764896.368134:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.368136:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.368155:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100577280 i_blocks 90112 00000080:00200000:0.0:1775764896.368158:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.368168:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100577280, count: 4096 00000080:00200000:0.0:1775764896.368174:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100577280, bytes: 4096 00000080:00200000:0.0:1775764896.368176:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.368178:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24556, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.368179:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100577280, 100581376) 0 0 00000080:00200000:0.0:1775764896.368181:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.368184:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24555, 24555] 00020000:00200000:0.0:1775764896.368186:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100577280 chunk: [100577280, 100581376) 100581376, 4096 00020000:00200000:0.0:1775764896.368188:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.368194:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24555, 24555] 00000008:00010000:0.0:1775764896.368198:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.368199:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.368204:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368210:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100577280 100581375) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368216:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368220:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368224:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.368228:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100577280, 100581376) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.368231:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24555 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.368244:0:18302:0:(rw26.c:858:ll_write_end()) pos 100577280, len 4096, copied 4096 00000080:00200000:0.0:1775764896.368248:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000c5d1cfd5[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.368251:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @0000000085b9fbcc 17ffffc0001001 3:0 ffff90c212838a80 24555 lru 00000080:00200000:0.0:1775764896.368257:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000002c0b41a9 24555: 1< 0 - - > 2< 100577280 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 36864 - | + - + - > 5< + - + - | 0 - | 4075 - -> 00000080:00200000:0.0:1775764896.368259:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000c5d1cfd5 00000080:00200000:0.0:1775764896.368260:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.368262:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.368270:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368278:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368284:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368292:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.368303:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.368305:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.368307:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.368314:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.368316:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.368318:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.368320:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368325:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.368330:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.368331:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.368334:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.368336:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24555, 24555] 00000080:00200000:0.0:1775764896.368343:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.368345:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.368363:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100581376 i_blocks 90112 00000080:00200000:0.0:1775764896.368366:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.368376:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100581376, count: 4096 00000080:00200000:0.0:1775764896.368382:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100581376, bytes: 4096 00000080:00200000:0.0:1775764896.368384:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.368386:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24557, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.368387:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100581376, 100585472) 0 0 00000080:00200000:0.0:1775764896.368389:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.368392:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24556, 24556] 00020000:00200000:0.0:1775764896.368394:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100581376 chunk: [100581376, 100585472) 100585472, 4096 00020000:00200000:0.0:1775764896.368396:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.368402:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24556, 24556] 00000008:00010000:0.0:1775764896.368405:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.368407:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.368412:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368418:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100581376 100585471) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368423:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368428:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368431:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.368437:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100581376, 100585472) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.368440:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24556 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.368452:0:18302:0:(rw26.c:858:ll_write_end()) pos 100581376, len 4096, copied 4096 00000080:00200000:0.0:1775764896.368455:0:18302:0:(rw26.c:916:ll_write_end()) page@000000001f386b8d[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.368457:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @00000000f43fef96 17ffffc0001001 3:0 ffff90c212838700 24556 lru 00000080:00200000:0.0:1775764896.368460:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000ecd4a65d 24556: 1< 0 - - > 2< 100581376 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 32768 - | + - + - > 5< + - + - | 0 - | 4076 - -> 00000080:00200000:0.0:1775764896.368462:0:18302:0:(rw26.c:916:ll_write_end()) end page@000000001f386b8d 00000080:00200000:0.0:1775764896.368463:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.368464:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.368524:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368530:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368534:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368539:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.368546:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.368548:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.368550:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.368556:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.368558:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.368559:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.368562:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368567:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.368573:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.368574:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.368577:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.368579:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24556, 24556] 00000080:00200000:0.0:1775764896.368586:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.368588:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.368606:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100585472 i_blocks 90112 00000080:00200000:0.0:1775764896.368609:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.368619:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100585472, count: 4096 00000080:00200000:0.0:1775764896.368625:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100585472, bytes: 4096 00000080:00200000:0.0:1775764896.368627:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.368630:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24558, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.368630:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100585472, 100589568) 0 0 00000080:00200000:0.0:1775764896.368632:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.368636:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24557, 24557] 00020000:00200000:0.0:1775764896.368638:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100585472 chunk: [100585472, 100589568) 100589568, 4096 00020000:00200000:0.0:1775764896.368639:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.368645:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24557, 24557] 00000008:00010000:0.0:1775764896.368649:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.368650:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.368655:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368661:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100585472 100589567) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368667:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368671:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368675:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.368679:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100585472, 100589568) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.368682:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24557 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.368695:0:18302:0:(rw26.c:858:ll_write_end()) pos 100585472, len 4096, copied 4096 00000080:00200000:0.0:1775764896.368697:0:18302:0:(rw26.c:916:ll_write_end()) page@0000000080562e22[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.368699:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000008b19a33e 17ffffc0001001 3:0 ffff90c212838c40 24557 lru 00000080:00200000:0.0:1775764896.368703:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@00000000c3443f62 24557: 1< 0 - - > 2< 100585472 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 28672 - | + - + - > 5< + - + - | 0 - | 4077 - -> 00000080:00200000:0.0:1775764896.368704:0:18302:0:(rw26.c:916:ll_write_end()) end page@0000000080562e22 00000080:00200000:0.0:1775764896.368705:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.368706:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00000008:00010000:0.0:1775764896.368711:0:18302:0:(osc_cache.c:564:osc_extent_merge()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368716:0:18302:0:(osc_cache.c:496:__osc_extent_remove()) ### extent: 000000005f816856 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368720:0:18302:0:(osc_cache.c:581:osc_extent_merge()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00010000:0.0:1775764896.368728:0:18302:0:(osc_cache.c:836:osc_extent_find()) ### extent: 000000002658a68e ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.368735:0:18302:0:(vvp_io.c:1080:vvp_set_batch_dirty()) mapping ffff90c21316b3f8, count 1, dirtied 1 00000080:00200000:0.0:1775764896.368736:0:18302:0:(vvp_io.c:1190:vvp_io_write_commit()) Committed 1 pages 4096 bytes, tot: 4096 00000080:00200000:0.0:1775764896.368738:0:18302:0:(vvp_io.c:1353:vvp_io_write_start()) f5.replay-ost-single: write bytes 0, result: 4096 00000020:00010000:0.0:1775764896.368744:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_release():245 00000020:00010000:0.0:1775764896.368745:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_cancel():142 00000020:00010000:0.0:1775764896.368747:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff90c2125aa4b0 (ffff90c210c811b8/1) at cl_lock_cancel():142 00010000:00010000:0.0:1775764896.368749:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368754:0:18302:0:(ldlm_lock.c:912:ldlm_lock_decref_internal()) ### add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,0 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000020:00010000:0.0:1775764896.368759:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2034a1d58 (ffff90c210c81878/0) at cl_lock_fini():72 00000020:00010000:0.0:1775764896.368761:0:18302:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff90c2125aa4b0 (ffff90c210c81878/1) at cl_lock_fini():72 00000080:00200000:0.0:1775764896.368763:0:18302:0:(vvp_io.c:517:vvp_io_advance()) advancing 4096 bytes 00000080:00200000:0.0:1775764896.368765:0:18302:0:(file.c:2173:ll_file_io_generic()) Range unlock [24557, 24557] 00000080:00200000:0.0:1775764896.368771:0:18302:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764896.368773:0:18302:0:(file.c:2206:ll_file_io_generic()) f5.replay-ost-single: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1775764896.368791:0:18302:0:(file.c:1742:ll_merge_attr_nolock()) [0x200000403:0x10:0x0] updating i_size 100589568 i_blocks 90112 00000080:00200000:0.0:1775764896.368794:0:18302:0:(file.c:2292:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1775764896.368804:0:18302:0:(file.c:2612:do_file_write_iter()) START file f5.replay-ost-single:[0x200000403:0x10:0x0], ppos: 100589568, count: 4096 00000080:00200000:0.0:1775764896.368810:0:18302:0:(file.c:2058:ll_file_io_generic()) f5.replay-ost-single: write ppos: 100589568, bytes: 4096 00000080:00200000:0.0:1775764896.368812:0:18302:0:(file.c:1815:ll_io_set_mirror()) f5.replay-ost-single: desiginated mirror: 0 00000020:00200000:0.0:1775764896.368814:0:18302:0:(cl_io.c:214:cl_io_rw_init()) header@0000000046dc5e4d[0x110, 24559, [0x200000403:0x10:0x0] hash] 00000020:00200000:0.0:1775764896.368815:0:18302:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [100589568, 100593664) 0 0 00000080:00200000:0.0:1775764896.368817:0:18302:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1775764896.368820:0:18302:0:(file.c:2136:ll_file_io_generic()) Range lock [24558, 24558] 00020000:00200000:0.0:1775764896.368822:0:18302:0:(lov_io.c:1052:lov_io_rw_iter_init()) stripe: 100589568 chunk: [100589568, 100593664) 100593664, 4096 00020000:00200000:0.0:1775764896.368824:0:18302:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764896.368830:0:18302:0:(vvp_io.c:198:vvp_io_one_lock_index()) lock: 1 [24558, 24558] 00000008:00010000:0.0:1775764896.368833:0:18302:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 0 00000008:00010000:0.0:1775764896.368835:0:18302:0:(osc_lock.c:1310:osc_lock_init()) ### lock ffff90c2125aa4b0, osc lock ffff90c218d4ddb0, flags 0x0 00010000:00010000:0.0:1775764896.368840:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 3 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368845:0:18302:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100589568 100593663) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 15/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368851:0:18302:0:(ldlm_lock.c:775:ldlm_lock_addref_internal_nolock()) ### ldlm_lock_addref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368855:0:18302:0:(ldlm_lock.c:829:ldlm_lock_decref_internal_nolock()) ### ldlm_lock_decref(PW) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 19/0,2 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764896.368859:0:18302:0:(ldlm_lock.c:919:ldlm_lock_decref_internal()) ### do not add lock into lru list ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 18/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x810020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000080:00200000:0.0:1775764896.368863:0:18302:0:(vvp_io.c:1286:vvp_io_write_start()) f5.replay-ost-single: write [100589568, 100593664) append 0 append lockpos 0 00000080:00200000:0.0:1775764896.368866:0:18302:0:(rw26.c:663:ll_write_begin()) Writing 24558 of 0 to 4096 bytes 00000080:00200000:0.0:1775764896.368970:0:18302:0:(rw26.c:858:ll_write_end()) pos 100589568, len 4096, copied 4096 00000080:00200000:0.0:1775764896.368973:0:18302:0:(rw26.c:916:ll_write_end()) page@00000000d5d86abc[3 00000000b99c1f5d 2 1 000000006eddc6d8] 00000080:00200000:0.0:1775764896.368975:0:18302:0:(rw26.c:916:ll_write_end()) vmpage @000000003d513208 17ffffc0001001 3:0 ffff90c2128388c0 24558 lru 00000080:00200000:0.0:1775764896.368979:0:18302:0:(rw26.c:916:ll_write_end()) osc-page@000000006f20cb3f 24558: 1< 0 - - > 2< 100589568 0 0 0x0 0x0 | 00000000fcbe25c6 000000002d2b543e > 3< 0 0 > 4< 0 9 8 24576 - | + - + - > 5< + - + - | 0 - | 4078 - -> 00000080:00200000:0.0:1775764896.368981:0:18302:0:(rw26.c:916:ll_write_end()) end page@00000000d5d86abc 00000080:00200000:0.0:1775764896.368981:0:18302:0:(rw26.c:916:ll_write_end()) queued cl_page: 1. 00000080:00200000:0.0:1775764896.368983:0:18302:0:(vvp_io.c:1167:vvp_io_write_commit()) commit async pages: 1, from 0, to 4096 prio 0 00010000:00010000:3.0:1775764896.392096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764896.903098:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c5000 lustre-OST0001_UUID: changing import state from FULL to CONNECTING 00000100:00080000:2.0:1775764896.903104:0:14408:0:(import.c:2095:ptlrpc_disconnect_and_idle_import()) lustre-OST0001-osc-ffff90c203172800: disconnect after 20s idle 00000100:00100000:3.0:1775764896.903141:0:2428:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8b4cb7d5-7b49-44b3-b066-941aae62e872:2428:1862023741807616:192.168.202.155@tcp:400:kworker.0 00000100:00100000:2.0:1775764896.903150:0:2427:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2427:1862023741807744:192.168.202.155@tcp:400:kworker.0 00000100:00100000:1.0:1775764896.903218:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741808128:192.168.202.155@tcp:9:kworker.0 00000100:00100000:3.0:1775764896.903308:0:2428:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2428:1862023741807872:192.168.202.155@tcp:400:kworker.0 00000100:00100000:2.0:1775764896.905762:0:2427:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2427:1862023741808000:192.168.202.155@tcp:400:kworker.0 00000100:00100000:3.0:1775764896.907685:0:2427:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2427:1862023741807744:192.168.202.155@tcp:400:kworker.0 00000100:00100000:2.0:1775764896.907701:0:2428:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8b4cb7d5-7b49-44b3-b066-941aae62e872:2428:1862023741807616:192.168.202.155@tcp:400:kworker.0 00010000:00080000:2.0:1775764896.907723:0:2428:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff90c2034be680 x1862023741807872/t0(0) o400->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 0 dl 1775764912 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764896.907737:0:2428:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2428:1862023741807872:192.168.202.155@tcp:400:kworker.0 00000100:00080000:3.0:1775764896.908339:0:2427:0:(recover.c:203:ptlrpc_request_handle_notconn()) import lustre-OST0000-osc-ffff90c203172800 of lustre-OST0000_UUID@192.168.202.155@tcp abruptly disconnected: reconnecting 00000100:00080000:3.0:1775764896.908347:0:2427:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:3.0:1775764896.908351:0:2427:0:(import.c:181:ptlrpc_set_import_discon()) lustre-OST0000-osc-ffff90c203172800: Connection to lustre-OST0000 (at 192.168.202.155@tcp) was lost; in progress operations using this service will wait for recovery to complete 00010000:00080000:1.0:1775764896.908379:0:2424:0:(ldlm_request.c:1663:ldlm_cli_update_pool()) @@@ Zero SLV or limit found (SLV=0, limit=61549) req@ffff90c2034bdf80 x1862023741808128/t0(0) o9->lustre-OST0001-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764906 ref 1 fl Rpc:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764896.908392:0:2424:0:(import.c:2005:ptlrpc_disconnect_idle_interpret()) @@@ inflight=1, refcount=3: rc = 0 req@ffff90c2034bdf80 x1862023741808128/t0(0) o9->lustre-OST0001-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 224/224 e 0 to 0 dl 1775764906 ref 1 fl Interpret:RNQU/200/0 rc 0/0 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764896.908400:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c5000 lustre-OST0001_UUID: changing import state from CONNECTING to IDLE 00000100:00100000:1.0:1775764896.908404:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741808128:192.168.202.155@tcp:9:kworker.0 00000100:00080000:3.0:1775764896.923531:0:2427:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764896.923547:0:2427:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 741 00000100:00080000:3.0:1775764896.923633:0:2427:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764896.923645:0:2427:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775764896.923678:0:2427:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741808256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764896.923772:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741808256:192.168.202.155@tcp:8:ptlrpcd_01_00.0 00000100:00100000:3.0:1775764896.924118:0:2427:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdc00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2427:1862023741808000:192.168.202.155@tcp:400:kworker.0 00000100:00080000:1.0:1775764896.926593:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764896.926605:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-19) 00000100:00100000:1.0:1775764896.926610:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741808256:192.168.202.155@tcp:8:ptlrpcd_01_00.0 00010000:00010000:1.0:1775764897.415242:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764898.438996:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764899.464946:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764900.487111:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764901.511127:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764902.023146:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775764902.023152:0:2425:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8b4cb7d5-7b49-44b3-b066-941aae62e872:2425:1862023741808384:192.168.202.155@tcp:400:kworker.0 00000100:00080000:0.0:1775764902.023153:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775764902.023158:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764902.023174:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 790 00000100:00080000:0.0:1775764902.023182:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1775764902.023199:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775764902.023209:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775764902.023236:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741808768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764902.023253:0:2425:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741808640:192.168.202.155@tcp:400:kworker.0 00000100:00100000:1.0:1775764902.023283:0:2425:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1775764902.023291:0:2425:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741808512:192.168.202.155@tcp:400:kworker.0 00000100:00100000:3.0:1775764902.023307:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741808768:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775764902.535093:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1775764903.047023:0:2427:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764887/real 1775764887] req@ffff90c2034bd880 x1862023741805952/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 488/448 e 0 to 1 dl 1775764903 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:3.0:1775764903.055667:0:2427:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff90c2034c7800 already not connected (conn 30, was 28): CONNECTING 00010000:00010000:1.0:1775764903.559223:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1775764904.006998:0:2428:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764888/real 1775764888] req@ffff90c218da1180 x1862023741806080/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 488/448 e 0 to 1 dl 1775764904 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1775764904.015158:0:2428:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff90c2034c7800 already not connected (conn 30, was 28): CONNECTING 00010000:00010000:0.0:1775764904.583004:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:3.0:1775764905.094990:0:2427:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764889/real 1775764889] req@ffff90c218da2a00 x1862023741806208/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 488/448 e 0 to 1 dl 1775764905 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:3.0:1775764905.102983:0:2427:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff90c2034c7800 already not connected (conn 30, was 28): CONNECTING 00010000:00010000:3.0:1775764905.607058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1775764906.054987:0:2428:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764890/real 1775764890] req@ffff90c218da1500 x1862023741806336/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 488/448 e 0 to 1 dl 1775764906 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1775764906.055040:0:2428:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff90c2034c7800 already not connected (conn 30, was 28): CONNECTING 00000100:00000400:0.0:1775764906.630989:0:2426:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764890/real 1775764890] req@ffff90c203638a80 x1862023741806464/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 488/448 e 0 to 1 dl 1775764906 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00010000:00010000:1.0:1775764906.631026:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764906.631046:0:2426:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff90c2034c7800 already not connected (conn 30, was 28): CONNECTING 00000100:00000400:2.0:1775764907.143024:0:2427:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764891/real 1775764891] req@ffff90c218da2680 x1862023741806592/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 488/448 e 0 to 1 dl 1775764907 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:0.0:1775764907.143086:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764907.143090:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1775764907.143109:0:2425:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741809024:192.168.202.155@tcp:400:kworker.0 00000100:00100000:0.0:1775764907.143196:0:2425:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [1->0] 00000100:00100000:0.0:1775764907.143203:0:2425:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8b4cb7d5-7b49-44b3-b066-941aae62e872:2425:1862023741808896:192.168.202.155@tcp:400:kworker.0 00000100:00100000:0.0:1775764907.143229:0:2425:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741809152:192.168.202.155@tcp:400:kworker.0 00000100:00080000:2.0:1775764907.153157:0:2427:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff90c2034c7800 already not connected (conn 30, was 28): CONNECTING 00000080:00200000:0.0:1775764907.655021:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655037:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00010000:00010000:3.0:1775764907.655043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1775764907.655103:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764907.655129:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655133:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764907.655156:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764907.655173:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655177:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764907.655197:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764907.655214:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655218:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764907.655238:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764907.655255:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655259:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764907.655277:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764907.655293:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655296:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764907.655313:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764907.655329:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655332:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764907.655351:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764907.655367:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655371:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764907.655390:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764907.655407:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655411:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764907.655425:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764907.655435:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764907.655437:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764907.655446:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00000400:1.0:1775764908.678948:0:2426:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764892/real 1775764892] req@ffff90c20363ad80 x1862023741807232/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 488/448 e 0 to 1 dl 1775764908 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:1.0:1775764908.678988:0:2426:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff90c2034c7800 already not connected (conn 30, was 28): CONNECTING 00010000:00010000:3.0:1775764908.678990:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1775764909.127006:0:2428:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764893/real 1775764893] req@ffff90c2034bc000 x1862023741807360/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 488/448 e 0 to 1 dl 1775764909 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1775764909.127055:0:2428:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff90c2034c7800 already not connected (conn 30, was 28): CONNECTING 00010000:00010000:1.0:1775764909.703033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1775764910.214973:0:2427:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764894/real 1775764894] req@ffff90c2034bf800 x1862023741807488/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 488/448 e 0 to 1 dl 1775764910 ref 2 fl Rpc:XQr/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00000100:00080000:2.0:1775764910.215022:0:2427:0:(import.c:213:ptlrpc_set_import_discon()) osc: import ffff90c2034c7800 already not connected (conn 30, was 28): CONNECTING 00010000:00010000:3.0:1775764910.727043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764911.751048:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764912.263010:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764912.263014:0:38:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1775764912.263034:0:2426:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2426:1862023741809408:192.168.202.155@tcp:400:kworker.0 00000100:00100000:1.0:1775764912.263082:0:2425:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8b4cb7d5-7b49-44b3-b066-941aae62e872:2425:1862023741809280:192.168.202.155@tcp:400:kworker.0 00000100:00100000:1.0:1775764912.263126:0:2425:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741809536:192.168.202.155@tcp:400:kworker.0 00000080:00200000:0.0:1775764912.775022:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775035:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00010000:00010000:3.0:1775764912.775040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:0.0:1775764912.775084:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775106:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775110:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764912.775132:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775152:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775156:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764912.775176:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775192:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775195:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764912.775206:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775218:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775221:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764912.775237:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775253:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775258:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764912.775276:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775289:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775291:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764912.775302:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775310:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775312:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764912.775321:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775330:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775332:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764912.775340:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775349:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775350:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1775764912.775359:0:38:0:(vvp_io.c:288:vvp_io_fini()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1775764912.775371:0:38:0:(vvp_io.c:1817:vvp_io_init()) [0x200000403:0x10:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:0.0:1775764912.775373:0:38:0:(lov_io.c:860:lov_io_iter_init()) component[0] flags 0x10 00000100:00100000:0.0:1775764912.775433:0:2425:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1775764912.775449:0:2425:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff90c203639880 x1862023741809664/t0(0) o16->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 440/432 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1775764913.799049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764914.823053:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764915.847035:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764916.871038:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764917.382977:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764902/real 1775764902] req@ffff90c203638380 x1862023741808768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764917 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764917.383007:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741808768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764917 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764917.383025:0:2428:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8b4cb7d5-7b49-44b3-b066-941aae62e872:2428:1862023741809792:192.168.202.155@tcp:400:kworker.0 00000100:00080000:0.0:1775764917.383026:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764917.383031:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764917.383037:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764917.383040:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764917.383046:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764917.383050:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741808768:192.168.202.155@tcp:8:kworker.0 00000100:00080000:3.0:1775764917.383101:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 795 00000100:00080000:3.0:1775764917.383107:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 10s 00000100:00080000:3.0:1775764917.383113:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764917.383123:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775764917.383126:0:2428:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2428:1862023741810048:192.168.202.155@tcp:400:kworker.0 00000100:00100000:3.0:1775764917.383142:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741810176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775764917.383158:0:2428:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1775764917.383166:0:2428:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2428:1862023741809920:192.168.202.155@tcp:400:kworker.0 00000100:00100000:0.0:1775764917.383192:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741810176:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775764917.895015:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:0.0:1775764918.918974:0:2425:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764902/real 1775764902] req@ffff90c20363bb80 x1862023741808512/t0(0) o400->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764918 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1775764918.919013:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764918.929116:0:2425:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff90c20363bb80 x1862023741808512/t0(0) o400->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764918 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764918.929129:0:2425:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from FULL to DISCONN 00000100:02000400:0.0:1775764918.929133:0:2425:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0000-mdc-ffff90c203172800: Connection to lustre-MDT0000 (at 192.168.202.155@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:0.0:1775764918.933950:0:2425:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0000_UUID: waking up pinger s:DISCONN 00000100:00000400:0.0:1775764918.933982:0:2425:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764902/real 1775764902] req@ffff90c20363b100 x1862023741808640/t0(0) o400->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764918 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764918.934007:0:2425:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff90c20363b100 x1862023741808640/t0(0) o400->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764918 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764918.934015:0:2425:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from FULL to DISCONN 00000100:02000400:0.0:1775764918.934019:0:2425:0:(import.c:181:ptlrpc_set_import_discon()) lustre-MDT0001-mdc-ffff90c203172800: Connection to lustre-MDT0001 (at 192.168.202.155@tcp) was lost; in progress operations using this service will wait for recovery to complete 00000100:00080000:0.0:1775764918.934022:0:2425:0:(import.c:416:ptlrpc_pinger_force()) lustre-MDT0001_UUID: waking up pinger s:DISCONN 00000100:00000400:0.0:1775764918.934034:0:2425:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764902/real 1775764902] req@ffff90c203639180 x1862023741808384/t0(0) o400->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764918 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764918.934044:0:2425:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff90c203639180 x1862023741808384/t0(0) o400->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764918 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764918.934049:0:2425:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from FULL to DISCONN 00000100:02020000:0.0:1775764918.934051:0:2425:0:(import.c:185:ptlrpc_set_import_discon()) MGC192.168.202.155@tcp: Connection to MGS (at 192.168.202.155@tcp) was lost; in progress operations using this service will fail 00000100:00080000:3.0:1775764918.934054:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764918.934061:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764918.934064:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764918.934079:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 489 00000100:00080000:3.0:1775764918.934092:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764918.934101:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764918.934114:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741810432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764918.934131:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764918.934133:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764918.934134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764918.934137:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 489 00000100:00080000:3.0:1775764918.934140:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764918.934143:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764918.934149:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741810560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764918.934161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764918.934164:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1775764918.934167:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741810432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764918.934254:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741810560:192.168.202.155@tcp:38:kworker.0 00000100:00080000:0.0:1775764918.937144:0:2425:0:(import.c:191:ptlrpc_set_import_discon()) import MGS@192.168.202.155@tcp for MGC192.168.202.155@tcp not replayable, auto-deactivating 00000100:00080000:0.0:1775764918.937148:0:2425:0:(import.c:138:ptlrpc_deactivate_import_nolock()) setting import MGS INVALID 00000100:00100000:0.0:1775764918.937154:0:2425:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff90c203639180 x1862023741808384/t0(0) o400->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 224/224 e 0 to 1 dl 1775764918 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764918.937165:0:2425:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff90c20363a680 x1862023741808896/t0(0) o400->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764923 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764918.937173:0:2425:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff90c20363aa00 x1862023741809280/t0(0) o400->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764928 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764918.937181:0:2425:0:(client.c:3560:ptlrpc_abort_inflight()) @@@ inflight req@ffff90c218da1880 x1862023741809792/t0(0) o400->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 224/224 e 0 to 0 dl 1775764933 ref 1 fl Rpc:NQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775764918.937193:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 10000000:01000000:0.0:1775764918.937195:0:2425:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808001 00000100:00080000:0.0:1775764918.937199:0:2425:0:(pinger.c:192:ptlrpc_pinger_ir_down()) IR down 10000000:01000000:0.0:1775764918.937201:0:2425:0:(mgc_request.c:1166:mgc_import_event()) import event 0x808002 00000100:00080000:3.0:1775764918.937202:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1775764918.937203:0:2425:0:(import.c:416:ptlrpc_pinger_force()) MGS: waking up pinger s:DISCONN 00000100:00100000:2.0:1775764918.937209:0:2427:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:8b4cb7d5-7b49-44b3-b066-941aae62e872:0:1862023741810304:192.168.202.155@tcp:400:kworker.0 00000100:00080000:3.0:1775764918.937220:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764918.937232:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764918.937234:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775764918.937237:0:2425:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8b4cb7d5-7b49-44b3-b066-941aae62e872:2425:1862023741809280:192.168.202.155@tcp:400:kworker.0 00000100:00080000:3.0:1775764918.937243:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 488 00000100:00080000:3.0:1775764918.937259:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775764918.937270:0:2425:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8b4cb7d5-7b49-44b3-b066-941aae62e872:2425:1862023741808896:192.168.202.155@tcp:400:kworker.0 00000100:00100000:0.0:1775764918.937281:0:2425:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741808512:192.168.202.155@tcp:400:kworker.0 00000100:00100000:3.0:1775764918.937281:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741810688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764918.937294:0:2425:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741808640:192.168.202.155@tcp:400:kworker.0 00000100:00100000:0.0:1775764918.937304:0:2425:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8b4cb7d5-7b49-44b3-b066-941aae62e872:2425:1862023741808384:192.168.202.155@tcp:400:kworker.0 00000100:00080000:3.0:1775764918.937310:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764918.937314:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1775764918.937326:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741810688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775764918.937353:0:2428:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8b4cb7d5-7b49-44b3-b066-941aae62e872:2428:1862023741809792:192.168.202.155@tcp:400:kworker.0 00010000:00010000:3.0:1775764919.943044:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764920.967012:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764921.991008:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764921.991018:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764921.991021:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764921.991025:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1775764921.991028:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764921.991093:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764921.991096:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764921.991098:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764921.991100:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:0.0:1775764923.014964:0:2425:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764907/real 1775764907] req@ffff90c203638700 x1862023741809152/t0(0) o400->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764923 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764923.014992:0:2425:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff90c203638700 x1862023741809152/t0(0) o400->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764923 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764923.015005:0:2425:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff90c21959b000 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:0.0:1775764923.015011:0:2425:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764907/real 1775764907] req@ffff90c203638000 x1862023741809024/t0(0) o400->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764923 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764923.015021:0:2425:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff90c203638000 x1862023741809024/t0(0) o400->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764923 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764923.015026:0:2425:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff90c219571800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:0.0:1775764923.015043:0:2425:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741809152:192.168.202.155@tcp:400:kworker.0 00000100:00100000:0.0:1775764923.015049:0:2425:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741809024:192.168.202.155@tcp:400:kworker.0 00010000:00010000:3.0:1775764923.015093:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764924.038973:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764924.038982:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764924.038985:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:2.0:1775764924.038990:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764924.038990:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764924.038993:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764924.038996:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764924.038999:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764924.039001:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:1.0:1775764925.063006:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764926.087006:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764927.111005:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764928.070960:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764918/real 1775764918] req@ffff90c218da0a80 x1862023741810688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764928.070993:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741810688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764928.071005:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764918/real 1775764918] req@ffff90c218da3800 x1862023741810432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764928.071017:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741810432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764928.071027:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764918/real 1775764918] req@ffff90c218da3480 x1862023741810560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764928.071036:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741810560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764928.071058:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764928.071086:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764928.071091:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741810688:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775764928.071100:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764928.071106:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764928.071110:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741810432:192.168.202.155@tcp:38:kworker.0 00000100:00080000:1.0:1775764928.071116:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775764928.071121:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764928.071125:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764928.071127:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764928.071133:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00080000:3.0:1775764928.071134:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 812 00000100:00100000:1.0:1775764928.071143:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741810560:192.168.202.155@tcp:38:kworker.0 00000100:00080000:3.0:1775764928.071143:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1775764928.071166:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775764928.071200:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741810816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764928.071216:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764928.071218:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764928.071220:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764928.071222:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 812 00000100:00080000:3.0:1775764928.071225:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1775764928.071227:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764928.071234:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764928.071239:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741810816:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775764928.071245:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741810944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764928.071252:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764928.071254:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764928.071255:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764928.071257:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 812 00000100:00080000:3.0:1775764928.071259:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 5s 00000100:00080000:3.0:1775764928.071261:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764928.071263:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764928.071270:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741811072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764928.071276:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764928.071279:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764928.071285:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764928.071286:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1775764928.071335:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741810944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764928.071360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811072:192.168.202.155@tcp:38:kworker.0 00000100:00000400:0.0:1775764928.134982:0:2425:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764912/real 1775764912] req@ffff90c20363a300 x1862023741809536/t0(0) o400->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1775764928.135011:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775764928.145373:0:2425:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff90c20363a300 x1862023741809536/t0(0) o400->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764928.145384:0:2425:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff90c21959b000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:0.0:1775764928.145397:0:2425:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2425:1862023741809536:192.168.202.155@tcp:400:kworker.0 00000100:00000400:1.0:1775764928.646980:0:2426:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764912/real 1775764912] req@ffff90c203639500 x1862023741809408/t0(0) o400->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764928.647007:0:2426:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff90c203639500 x1862023741809408/t0(0) o400->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764928 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764928.647013:0:2426:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff90c219571800 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1775764928.647026:0:2426:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2426:1862023741809408:192.168.202.155@tcp:400:kworker.0 00010000:00010000:3.0:1775764929.159033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764930.183020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764931.207026:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764932.231006:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764933.254939:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764933.254948:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764933.254951:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764933.254955:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764933.254958:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764933.254963:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:3.0:1775764933.254966:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764933.254969:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1775764933.255001:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1775764933.766957:0:2428:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764917/real 1775764917] req@ffff90c218da0700 x1862023741809920/t0(0) o400->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764933 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764933.766987:0:2428:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff90c218da0700 x1862023741809920/t0(0) o400->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764933 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764933.766996:0:2428:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff90c219571800 already not connected (conn 3, was 1): CONNECTING 00000100:00000400:2.0:1775764933.767003:0:2428:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764917/real 1775764917] req@ffff90c218da3100 x1862023741810048/t0(0) o400->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764933 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775764933.767015:0:2428:0:(client.c:2523:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff90c218da3100 x1862023741810048/t0(0) o400->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 224/224 e 0 to 1 dl 1775764933 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775764933.767025:0:2428:0:(import.c:213:ptlrpc_set_import_discon()) mdc: import ffff90c21959b000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:2.0:1775764933.767037:0:2428:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2428:1862023741809920:192.168.202.155@tcp:400:kworker.0 00000100:00100000:2.0:1775764933.767041:0:2428:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2428:1862023741810048:192.168.202.155@tcp:400:kworker.0 00010000:00010000:1.0:1775764934.279016:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764935.303127:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764936.327115:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764937.286915:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764917/real 1775764917] req@ffff90c218da1f80 x1862023741810176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764937 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764937.286944:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741810176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764937 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764937.286962:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764937.286979:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764937.286982:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741810176:192.168.202.155@tcp:8:kworker.0 00000100:00080000:2.0:1775764937.287026:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764937.287030:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764937.287032:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764937.287034:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764937.287036:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764937.287037:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764937.287039:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764937.287041:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775764937.287043:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764937.287048:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 810 00000100:00080000:2.0:1775764937.287056:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1775764937.287060:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775764937.287068:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775764937.287090:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be680 x1862023741811200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764937.287141:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811200:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775764937.351021:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764938.375008:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764939.399008:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764940.423019:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764941.447002:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775764942.470941:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764942.470947:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764942.470950:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764942.470952:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764942.470954:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764942.470956:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1775764942.470958:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764942.470959:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1775764942.471028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1775764942.598970:0:2416:0:(socklnd_cb.c:2376:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.202.155@tcp (192.168.202.155:988) timed out; the network or node may be down. 00000100:00100000:1.0:1775764943.430944:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764928/real 1775764928] req@ffff90c218da0000 x1862023741810944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764943 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764943.430968:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741810944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764943 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764943.430984:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764928/real 1775764928] req@ffff90c218da3480 x1862023741811072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764943 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764943.430996:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741811072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764943 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764943.431006:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764928/real 1775764928] req@ffff90c218da0380 x1862023741810816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764943 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764943.431017:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741810816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764943 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764943.431034:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764943.431052:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764943.431055:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741810944:192.168.202.155@tcp:38:kworker.0 00000100:00080000:1.0:1775764943.431058:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764943.431062:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764943.431064:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811072:192.168.202.155@tcp:38:kworker.0 00000100:00080000:1.0:1775764943.431066:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764943.431069:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764943.431070:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741810816:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775764943.431108:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764943.431113:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775764943.431115:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764943.431120:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 821 00000100:00080000:2.0:1775764943.431128:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1775764943.431132:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775764943.431158:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bed80 x1862023741811328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764943.431175:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764943.431177:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775764943.431178:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764943.431181:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 821 00000100:00080000:2.0:1775764943.431183:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1775764943.431185:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775764943.431192:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764943.431199:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023741811456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764943.431206:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764943.431209:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775764943.431211:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741811328:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775764943.431211:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775764943.431218:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 821 00000100:00080000:2.0:1775764943.431221:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1775764943.431223:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775764943.431225:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775764943.431234:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023741811584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775764943.431243:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775764943.431246:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:1.0:1775764943.431299:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764943.431330:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811584:192.168.202.155@tcp:38:kworker.0 00010000:00010000:1.0:1775764943.495062:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764944.519039:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775764945.543184:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.155:988 00010000:00010000:0.0:1775764945.543206:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775764945.543238:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775764945.543253:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775764945.543290:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775764945.543299:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:1.0:1775764945.543367:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764943/real 1775764945] req@ffff90c2034bf100 x1862023741811456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764945.543406:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023741811456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764945.543418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764945.543422:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764945.543426:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764945.543435:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764943/real 1775764945] req@ffff90c2034bc700 x1862023741811584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764945.543448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023741811584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764945.543455:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764945.543459:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764945.543462:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764945.543467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764943/real 1775764945] req@ffff90c2034bed80 x1862023741811328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764945.543478:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bed80 x1862023741811328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764945.543484:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764945.543487:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764945.543490:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741811328:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775764946.567052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00010000:0.0:1775764947.079030:0:18302:0:(osc_cache.c:2998:osc_queue_sync_pages()) ### no dlm lock: extent: ffff90c21871a540 00000008:00010000:0.0:1775764947.079095:0:6628:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff90c21871a540 00000008:00010000:0.0:1775764947.079102:0:6628:0:(osc_cache.c:2012:try_to_add_extent_for_io()) ### extent: 00000000bb73c2e8 ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 16/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00010000:00010000:0.0:1775764947.079146:0:6628:0:(ldlm_lock.c:1522:ldlm_lock_match_with_skip()) ### matched (100589568 100593663) ns: lustre-OST0000-osc-ffff90c203172800 lock: 0000000056e6c7b9/0xfb95bb0df4166db4 lrc: 17/0,1 mode: PW/PW res: [0x280000401:0x11:0x0].0x0 rrc: 2 type: EXT [0->18446744073709551615] (req 0->4095) gid 0 flags: 0x800020000000000 nid: local remote: 0x721ffa0b6629a7b0 expref: -99 pid: 18302 timeout: 0 lvb_type: 1 lru_score: 254 lru_type: 1 00000008:00100000:0.0:1775764947.079239:0:6628:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff90c20363a300 - object 0x280000401:17 offset 100589568<>100593664 00000100:00080000:0.0:1775764947.079312:0:2425:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff90c20363a300 x1862023741811712/t0(0) o4->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:6/4 lens 4584/448 e 0 to 0 dl 0 ref 2 fl Rpc:WQU/600/ffffffff rc 0/-1 job:'iozone.0' uid:0 gid:0 projid:0 00010000:00010000:3.0:1775764947.591055:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764948.614962:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764948.614972:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764948.614976:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764948.614987:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 837 00000100:00080000:3.0:1775764948.614998:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764948.615003:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775764948.615040:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741811840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764948.615063:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764948.615067:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764948.615068:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764948.615073:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 837 00000100:00080000:3.0:1775764948.615078:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764948.615081:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764948.615090:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764948.615104:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741811968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764948.615113:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741811840:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775764948.615116:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764948.615122:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764948.615124:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764948.615128:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 837 00000100:00080000:3.0:1775764948.615133:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 15s 00000100:00080000:3.0:1775764948.615137:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764948.615142:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764948.615151:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741812096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764948.615158:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764948.615161:0:14408:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1775764948.615184:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764948.615214:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764948.615245:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812096:192.168.202.155@tcp:38:kworker.0 00010000:00010000:1.0:1775764949.639026:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764950.663017:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775764951.687132:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.155:988 00010000:00010000:0.0:1775764951.687160:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775764951.687175:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775764951.687191:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775764951.687222:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775764951.687230:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:1.0:1775764951.687271:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764948/real 1775764951] req@ffff90c218da0a80 x1862023741811968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764951.687301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741811968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764951.687311:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764951.687314:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764951.687317:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764951.687322:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764948/real 1775764951] req@ffff90c218da1880 x1862023741812096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764951.687330:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741812096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764951.687334:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764951.687336:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764951.687338:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764951.687341:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764948/real 1775764951] req@ffff90c218da3800 x1862023741811840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764951.687349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741811840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764951.687356:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764951.687359:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764951.687362:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741811840:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775764952.711071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764953.734995:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764953.735005:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764953.735009:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764953.735021:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 842 00000100:00080000:3.0:1775764953.735035:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764953.735044:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:2.0:1775764953.735094:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764953.735094:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741812224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764953.735120:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764953.735126:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764953.735128:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764953.735134:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 842 00000100:00080000:3.0:1775764953.735138:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764953.735142:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764953.735153:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764953.735157:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741812224:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775764953.735165:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741812352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764953.735174:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764953.735177:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764953.735179:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764953.735181:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 842 00000100:00080000:3.0:1775764953.735183:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764953.735186:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764953.735190:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764953.735201:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741812480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764953.735208:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764953.735211:0:7481:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1775764953.735289:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764953.735325:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812480:192.168.202.155@tcp:38:kworker.0 00010000:00010000:3.0:1775764954.759229:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775764954.759246:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.155:988 00000400:00000100:0.0:1775764954.759288:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775764954.759303:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775764954.759341:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775764954.759349:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:1.0:1775764954.759395:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764953/real 1775764954] req@ffff90c218da0e00 x1862023741812352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764954.759426:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741812352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764954.759436:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764954.759441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764954.759444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764954.759453:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764953/real 1775764954] req@ffff90c218da3800 x1862023741812480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764954.759465:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741812480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764954.759471:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764954.759474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764954.759477:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764954.759483:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764953/real 1775764954] req@ffff90c218da2d80 x1862023741812224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764954.759494:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741812224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764954.759500:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764954.759503:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764954.759507:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741812224:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775764955.783116:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775764956.807020:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764957.831074:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775764958.854952:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764958.854962:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775764958.854966:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764958.854976:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 847 00000100:00080000:1.0:1775764958.854986:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764958.854993:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775764958.855000:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775764958.855026:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741812608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764958.855047:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764958.855050:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775764958.855052:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764958.855055:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 847 00000100:00080000:1.0:1775764958.855058:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764958.855061:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775764958.855069:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764958.855079:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741812736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764958.855086:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741812608:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775764958.855088:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764958.855093:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775764958.855094:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775764958.855097:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 847 00000100:00080000:1.0:1775764958.855099:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775764958.855101:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775764958.855105:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764958.855111:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741812864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775764958.855118:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775764958.855120:0:7481:0:(pinger.c:247:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1775764958.855166:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764958.855188:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812864:192.168.202.155@tcp:38:kworker.0 00010000:00010000:3.0:1775764959.879040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764960.902992:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764961.927061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775764961.927127:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.155:988 00000400:00000100:1.0:1775764961.927164:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775764961.927179:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775764961.927202:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775764961.927208:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775764961.927232:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764958/real 1775764961] req@ffff90c203496a00 x1862023741812736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764961.927258:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741812736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764961.927269:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764961.927274:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764961.927278:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764961.927286:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764958/real 1775764961] req@ffff90c203496300 x1862023741812864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764961.927297:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741812864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764961.927304:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764961.927307:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764961.927310:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741812864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764961.927316:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764958/real 1775764961] req@ffff90c203497b80 x1862023741812608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764961.927327:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741812608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764961.927334:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764961.927337:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764961.927340:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741812608:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775764962.950950:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1775764937/real 1775764937] req@ffff90c2034be680 x1862023741811200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764962 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764962.950980:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be680 x1862023741811200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764962 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1775764962.950980:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764962.951001:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764962.951019:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764962.951023:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741811200:192.168.202.155@tcp:8:kworker.0 00000100:00080000:3.0:1775764962.951042:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764962.951048:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764962.951052:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764962.951063:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 852 00000100:00080000:3.0:1775764962.951072:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764962.951079:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775764962.951107:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741812992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764962.951137:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764962.951141:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764962.951142:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764962.951146:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 852 00000100:00080000:3.0:1775764962.951150:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764962.951154:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775764962.951176:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741812992:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775764962.951177:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764962.951197:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741813120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764962.951208:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764962.951211:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764962.951213:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764962.951217:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 852 00000100:00080000:3.0:1775764962.951221:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764962.951225:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764962.951230:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764962.951241:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741813248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775764962.951245:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775764962.951254:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764962.951257:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764962.951259:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764962.951275:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 830 00000100:00080000:3.0:1775764962.951280:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764962.951284:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775764962.951290:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764962/real 1775764962] req@ffff90c218da1880 x1862023741812992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775764962.951291:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775764962.951306:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741813376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764962.951311:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741812992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764962.951332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764962.951337:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764962.951340:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741812992:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775764962.951358:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764962.951388:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764962.951400:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764962/real 1775764962] req@ffff90c218da0a80 x1862023741813120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764962.951411:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741813120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764962.951425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764962.951428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764962.951431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764962.951437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764962.951455:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764962.951464:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764962/real 1775764962] req@ffff90c218da0380 x1862023741813248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764962.951475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741813248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764962.951482:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764962.951486:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764962.951489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764962.952211:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813376:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775764962.952252:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764962.952269:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764962/real 1775764962] req@ffff90c218da3480 x1862023741813376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764987 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764962.952282:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741813376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764987 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764962.952291:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764962.952294:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764962.952298:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813376:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775764963.975057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764964.999010:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764966.023049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764967.047048:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764968.070973:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764968.070987:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764968.070993:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764968.071005:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 856 00000100:00080000:3.0:1775764968.071026:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764968.071034:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775764968.071043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764968.071077:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741813504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764968.071103:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764968.071106:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764968.071108:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764968.071111:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 856 00000100:00080000:3.0:1775764968.071114:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764968.071116:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764968.071125:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764968.071138:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741813632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764968.071151:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775764968.071156:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741813504:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775764968.071157:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764968.071161:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764968.071168:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 856 00000100:00080000:3.0:1775764968.071172:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764968.071177:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764968.071184:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764968.071199:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741813760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764968.071208:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764968.071210:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764968.071212:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764968.071216:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 856 00000100:00080000:3.0:1775764968.071219:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764968.071222:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764968.071227:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775764968.071228:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775764968.071245:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741813888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764968.071269:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764968/real 1775764968] req@ffff90c218da0000 x1862023741813504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764968.071285:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741813504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764968.071294:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764968.071299:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764968.071302:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741813504:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775764968.071323:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764968.071337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764968.071345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764968/real 1775764968] req@ffff90c218da1f80 x1862023741813632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764968.071353:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741813632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764968.071359:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764968.071361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764968.071363:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764968.071367:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764968.071385:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764968.071396:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764968/real 1775764968] req@ffff90c218da3100 x1862023741813760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764968.071409:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741813760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764968.071418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764968.071422:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764968.071425:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764968.071432:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813888:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775764968.071469:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764968.071479:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764968/real 1775764968] req@ffff90c218da0700 x1862023741813888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764968.071493:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741813888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764968.071504:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764968.071506:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764968.071509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741813888:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775764969.095043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764970.119025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764971.143003:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764972.167026:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764973.190982:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764973.190990:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764973.190993:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764973.191001:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 861 00000100:00080000:3.0:1775764973.191012:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764973.191019:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775764973.191058:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741814016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775764973.191081:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764973.191085:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764973.191090:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764973.191092:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764973.191097:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 861 00000100:00080000:3.0:1775764973.191102:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764973.191106:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764973.191119:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775764973.191135:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741814016:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775764973.191135:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741814144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764973.191151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764973.191155:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764973.191157:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764973.191161:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 861 00000100:00080000:3.0:1775764973.191164:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764973.191167:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764973.191171:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764973.191180:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741814272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764973.191186:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764973.191188:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764973.191189:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764973.191192:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 861 00000100:00080000:3.0:1775764973.191194:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764973.191196:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764973.191199:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775764973.191205:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741814400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764973.191240:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764973.191278:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764973.191304:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814400:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775764974.215148:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764975.239181:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775764976.263157:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.155:988 00000400:00000100:3.0:1775764976.263196:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775764976.263212:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00010000:00010000:0.0:1775764976.263241:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:3.0:1775764976.263255:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775764976.263264:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775764976.263272:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:1.0:1775764976.263311:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764973/real 1775764976] req@ffff90c218da3100 x1862023741814144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764976.263340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741814144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764976.263358:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764976.263363:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764976.263367:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764976.263407:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764973/real 1775764976] req@ffff90c218da1f80 x1862023741814272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764976.263421:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741814272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775764998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764976.263428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764976.263432:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764976.263435:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764976.263442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764973/real 1775764976] req@ffff90c218da0000 x1862023741814400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764976.263453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741814400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775764998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764976.263463:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764976.263466:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764976.263470:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814400:192.168.202.155@tcp:8:kworker.0 00000100:00100000:1.0:1775764976.263476:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764973/real 1775764976] req@ffff90c218da0700 x1862023741814016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764976.263487:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741814016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775764998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764976.263494:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764976.263497:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764976.263499:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741814016:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775764977.287066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764978.310967:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764978.310976:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775764978.310979:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764978.310986:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 866 00000100:00080000:0.0:1775764978.310995:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764978.311000:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775764978.311029:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741814528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775764978.311050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775764978.311055:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764978.311059:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775764978.311065:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764978.311069:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 866 00000100:00080000:0.0:1775764978.311073:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764978.311076:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775764978.311085:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764978.311097:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741814656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764978.311103:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741814528:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775764978.311113:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764978.311116:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775764978.311118:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764978.311122:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 866 00000100:00080000:0.0:1775764978.311126:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775764978.311130:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775764978.311135:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764978.311146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741814784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775764978.311156:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775764978.311159:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775764978.311161:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775764978.311165:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 866 00000100:00080000:0.0:1775764978.311168:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775764978.311169:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775764978.311176:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775764978.311181:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775764978.311193:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741814912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775764978.311213:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764978/real 1775764978] req@ffff90c203638000 x1862023741814528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764978.311233:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741814528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764978.311246:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764978.311250:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764978.311255:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741814528:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775764978.311280:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775764978.311302:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775764978.311314:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764978/real 1775764978] req@ffff90c203638700 x1862023741814656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764978.311326:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741814656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764978.311334:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764978.311338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764978.311341:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764978.311347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775764978.311381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775764978.311391:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764978/real 1775764978] req@ffff90c203639180 x1862023741814784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764978.311404:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741814784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764978.311412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764978.311415:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764978.311418:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775764978.311424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814912:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775764978.311442:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775764978.311451:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764978/real 1775764978] req@ffff90c20363b100 x1862023741814912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775764978.311462:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741814912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775764978.311472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775764978.311475:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775764978.311478:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741814912:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775764979.335146:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764980.358984:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764981.383019:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764982.407064:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764983.430958:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764983.430966:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764983.430970:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764983.430977:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 871 00000100:00080000:3.0:1775764983.430986:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764983.430991:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775764983.431020:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741815040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775764983.431031:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764983.431040:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764983.431043:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764983.431045:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764983.431047:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 871 00000100:00080000:3.0:1775764983.431050:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764983.431053:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764983.431063:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764983.431071:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741815040:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775764983.431077:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741815168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764983.431088:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764983.431091:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764983.431093:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764983.431097:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 871 00000100:00080000:3.0:1775764983.431101:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764983.431105:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764983.431110:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764983.431122:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741815296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775764983.431128:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775764983.431137:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764983.431141:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764983.431143:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764983.431148:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 871 00000100:00080000:3.0:1775764983.431152:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764983.431156:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775764983.431160:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764983/real 1775764983] req@ffff90c218da3480 x1862023741815040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775764983.431163:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775764983.431175:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741815424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764983.431184:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741815040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764983.431195:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764983.431200:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764983.431204:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741815040:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775764983.431225:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764983.431244:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764983.431252:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764983/real 1775764983] req@ffff90c218da0380 x1862023741815168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764983.431259:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741815168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764983.431265:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764983.431267:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764983.431269:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764983.431272:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764983.431281:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764983.431286:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764983/real 1775764983] req@ffff90c218da0a80 x1862023741815296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764983.431292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741815296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764983.431297:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764983.431298:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764983.431300:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764983.431303:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815424:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775764983.431326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764983.431331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764983/real 1775764983] req@ffff90c218da1880 x1862023741815424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764983.431338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741815424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764983.431344:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764983.431346:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764983.431348:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815424:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775764984.455010:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764985.479083:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764986.503031:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764987.527073:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764988.550930:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764988.550940:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764988.550944:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764988.550952:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 877 00000100:00080000:3.0:1775764988.550962:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764988.550967:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775764988.550997:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741815552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775764988.550998:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764988.551018:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764988.551020:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764988.551022:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764988.551024:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 877 00000100:00080000:3.0:1775764988.551026:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764988.551028:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764988.551035:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775764988.551040:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741815552:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775764988.551044:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741815680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764988.551052:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764988.551054:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764988.551055:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764988.551057:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 877 00000100:00080000:3.0:1775764988.551060:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764988.551062:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764988.551065:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764988.551073:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741815808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764988.551079:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764988.551081:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764988.551082:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764988.551084:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 877 00000100:00080000:3.0:1775764988.551086:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764988.551088:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:0.0:1775764988.551091:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775764988.551092:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775764988.551100:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741815936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764988.551121:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764988/real 1775764988] req@ffff90c218da2d80 x1862023741815552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764988.551134:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741815552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764988.551140:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764988.551143:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764988.551145:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741815552:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775764988.551165:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764988.551177:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764988.551183:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764988/real 1775764988] req@ffff90c218da3800 x1862023741815680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764988.551190:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741815680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764988.551194:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764988.551196:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764988.551198:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764988.551201:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815808:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764988.551210:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764988.551214:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764988/real 1775764988] req@ffff90c218da0e00 x1862023741815808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764988.551220:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741815808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764988.551224:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764988.551226:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764988.551227:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764988.551230:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815936:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775764988.551237:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764988.551242:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764988/real 1775764988] req@ffff90c218da1880 x1862023741815936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764988.551248:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741815936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764988.551253:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764988.551254:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764988.551256:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741815936:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775764989.575021:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775764990.598985:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764991.623123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764992.646978:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764993.670938:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764993.670946:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764993.670950:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764993.670957:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 882 00000100:00080000:3.0:1775764993.670969:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764993.670974:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775764993.671002:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775764993.671007:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741816064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764993.671030:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764993.671034:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764993.671036:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764993.671040:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 882 00000100:00080000:3.0:1775764993.671044:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764993.671048:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775764993.671054:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741816064:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775764993.671056:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764993.671071:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741816192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764993.671082:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764993.671084:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764993.671085:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764993.671088:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 882 00000100:00080000:3.0:1775764993.671092:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764993.671095:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764993.671098:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764993.671106:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741816320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764993.671112:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764993.671114:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764993.671115:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764993.671117:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 882 00000100:00080000:3.0:1775764993.671120:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764993.671121:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764993.671125:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775764993.671131:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741816448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764993.671158:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764993.671188:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764993.671203:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816448:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775764994.695069:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775764995.719154:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775764996.743201:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775764996.743209:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.202.155:988 00000400:00000100:3.0:1775764996.743250:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775764996.743266:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775764996.743303:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775764996.743317:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775764996.743327:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775764996.743341:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764993/real 1775764996] req@ffff90c218da0380 x1862023741816192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764996.743379:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741816192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764996.743394:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764996.743401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764996.743406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764996.743416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764993/real 1775764996] req@ffff90c218da3480 x1862023741816320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764996.743427:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741816320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764996.743432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764996.743435:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764996.743437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764996.743443:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764993/real 1775764996] req@ffff90c218da0700 x1862023741816448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764996.743454:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741816448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764996.743465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764996.743469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764996.743472:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816448:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775764996.743480:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764993/real 1775764996] req@ffff90c218da0a80 x1862023741816064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764996.743495:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741816064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764996.743502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764996.743505:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764996.743508:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741816064:192.168.202.155@tcp:250:kworker.0 00010000:00010000:0.0:1775764997.767095:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764998.790972:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764998.790979:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775764998.790982:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764998.790992:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 887 00000100:00080000:3.0:1775764998.791002:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764998.791009:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775764998.791040:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741816576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764998.791063:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764998.791066:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775764998.791067:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764998.791070:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 887 00000100:00080000:3.0:1775764998.791073:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775764998.791076:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764998.791084:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764998.791094:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741816704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764998.791102:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764998.791105:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775764998.791106:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764998.791113:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 887 00000100:00080000:3.0:1775764998.791117:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775764998.791118:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775764998.791120:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775764998.791129:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741816576:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775764998.791130:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775764998.791144:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741816832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775764998.791186:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775764998.791190:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775764998.791192:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775764998.791197:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 887 00000100:00080000:3.0:1775764998.791201:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775764998.791205:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775764998.791206:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775764998.791213:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775764998.791224:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741816960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775764998.791246:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764998/real 1775764998] req@ffff90c218da0000 x1862023741816576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764998.791264:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741816576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764998.791275:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764998.791278:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764998.791282:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741816576:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775764998.791288:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764998.791308:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764998.791318:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764998/real 1775764998] req@ffff90c218da1f80 x1862023741816704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764998.791328:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741816704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764998.791335:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764998.791338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764998.791341:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764998.791366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816832:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775764998.791381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764998.791389:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764998/real 1775764998] req@ffff90c218da3100 x1862023741816832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764998.791398:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741816832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764998.791406:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764998.791408:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764998.791411:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775764998.791416:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816960:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775764998.791431:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775764998.791440:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775764998/real 1775764998] req@ffff90c218da0a80 x1862023741816960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765023 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775764998.791456:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741816960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765023 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775764998.791466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775764998.791469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775764998.791472:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741816960:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775764999.815231:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765000.839036:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765001.863061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765002.886982:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765003.910948:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765003.910955:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765003.910959:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765003.910970:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 892 00000100:00080000:3.0:1775765003.910980:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775765003.911004:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765003.911017:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765003.911050:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741817088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765003.911069:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765003.911073:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765003.911074:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765003.911078:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 892 00000100:00080000:3.0:1775765003.911082:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765003.911086:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765003.911097:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741817088:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775765003.911106:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765003.911130:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741817216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765003.911140:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765003.911143:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765003.911145:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765003.911149:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 892 00000100:00080000:3.0:1775765003.911152:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765003.911155:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765003.911159:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765003.911169:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741817344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765003.911174:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765003.911182:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765003.911184:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765003.911186:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765003.911189:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 892 00000100:00080000:3.0:1775765003.911192:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765003.911194:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765003.911198:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765003.911205:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765003/real 1775765003] req@ffff90c218da0700 x1862023741817088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765003.911205:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741817472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765003.911221:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741817088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765003.911230:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765003.911234:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765003.911237:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741817088:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765003.911254:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817216:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765003.911269:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765003.911276:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765003/real 1775765003] req@ffff90c218da3480 x1862023741817216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765003.911288:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741817216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765003.911292:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765003.911294:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765003.911296:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765003.911299:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765003.911307:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765003.911312:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765003/real 1775765003] req@ffff90c218da0380 x1862023741817344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765003.911318:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741817344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765003.911322:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765003.911323:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765003.911325:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765003.911328:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817472:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765003.911335:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765003.911339:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765003/real 1775765003] req@ffff90c218da1880 x1862023741817472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765028 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765003.911345:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741817472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765028 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765003.911354:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765003.911355:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765003.911357:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817472:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765004.935077:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765005.959094:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765006.983042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765008.006967:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765009.031037:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765009.031047:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765009.031051:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765009.031059:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 897 00000100:00080000:1.0:1775765009.031077:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765009.031084:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765009.031094:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765009.031118:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741817600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765009.031139:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765009.031142:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765009.031143:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765009.031145:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 897 00000100:00080000:1.0:1775765009.031148:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765009.031150:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765009.031158:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765009.031165:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741817600:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765009.031167:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741817728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765009.031176:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765009.031178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765009.031179:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765009.031181:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 897 00000100:00080000:1.0:1775765009.031183:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765009.031185:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765009.031187:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765009.031195:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741817856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765009.031201:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765009.031203:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765009.031204:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765009.031206:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 897 00000100:00080000:1.0:1775765009.031208:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765009.031210:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765009.031212:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775765009.031217:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765009.031220:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741817984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765009.031250:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765009/real 1775765009] req@ffff90c203495880 x1862023741817600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765009.031262:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741817600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765009.031268:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765009.031271:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765009.031273:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741817600:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765009.031288:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765009.031300:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765009.031306:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765009/real 1775765009] req@ffff90c203495c00 x1862023741817728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765009.031313:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741817728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765009.031321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765009.031323:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765009.031325:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765009.031328:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765009.031336:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765009.031340:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765009/real 1775765009] req@ffff90c203494380 x1862023741817856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765009.031346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741817856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765009.031350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765009.031352:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765009.031353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765009.031356:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817984:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765009.031364:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765009.031368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765009/real 1775765009] req@ffff90c203496d80 x1862023741817984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765009.031374:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741817984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765009.031379:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765009.031381:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765009.031383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741817984:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765010.055123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765011.079059:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765012.103034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765013.127119:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765014.150978:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765014.150992:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765014.150999:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765014.151011:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 902 00000100:00080000:1.0:1775765014.151041:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765014.151054:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765014.151074:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765014.151103:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741818112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765014.151135:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765014.151138:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765014.151140:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765014.151146:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 902 00000100:00080000:1.0:1775765014.151149:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765014.151154:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765014.151166:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765014.151170:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741818112:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765014.151179:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741818240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765014.151190:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765014.151194:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765014.151196:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765014.151200:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 902 00000100:00080000:1.0:1775765014.151202:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765014.151205:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765014.151210:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765014.151218:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741818368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765014.151226:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765014.151229:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765014.151230:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765014.151233:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 902 00000100:00080000:1.0:1775765014.151236:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775765014.151242:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775765014.151248:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765014.151254:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765014.151276:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741818496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765014.151310:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765014/real 1775765014] req@ffff90c203497100 x1862023741818112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765014.151343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741818112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765014.151360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765014.151364:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765014.151367:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741818112:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765014.151388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818240:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765014.151408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765014.151424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765014/real 1775765014] req@ffff90c203494000 x1862023741818240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765014.151433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741818240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765014.151439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765014.151441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765014.151460:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765014.151466:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818368:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765014.151480:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765014.151486:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765014/real 1775765014] req@ffff90c203497480 x1862023741818368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765014.151494:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741818368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765014.151500:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765014.151502:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765014.151504:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765014.151508:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818496:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765014.151519:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765014.151524:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765014/real 1775765014] req@ffff90c203494e00 x1862023741818496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765014.151531:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741818496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765014.151538:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765014.151540:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765014.151542:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818496:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765015.175040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765016.199045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765017.223074:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765018.247090:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765019.270988:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765019.270996:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765019.271000:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765019.271008:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 907 00000100:00080000:3.0:1775765019.271024:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765019.271031:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765019.271064:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741818624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765019.271070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765019.271083:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765019.271086:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765019.271087:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765019.271091:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 907 00000100:00080000:3.0:1775765019.271094:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765019.271097:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765019.271105:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741818624:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775765019.271105:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765019.271116:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741818752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765019.271125:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765019.271128:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765019.271129:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765019.271131:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 907 00000100:00080000:3.0:1775765019.271134:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765019.271136:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765019.271139:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765019.271146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741818880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765019.271152:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765019.271154:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765019.271155:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765019.271158:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 907 00000100:00080000:3.0:1775765019.271162:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765019.271164:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765019.271167:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775765019.271172:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765019.271176:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741819008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765019.271209:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765019/real 1775765019] req@ffff90c218da0e00 x1862023741818624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765019.271223:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741818624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765019.271230:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765019.271233:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765019.271236:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741818624:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765019.271254:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765019.271268:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765019.271275:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765019/real 1775765019] req@ffff90c218da3800 x1862023741818752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765019.271283:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741818752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765019.271288:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765019.271290:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765019.271292:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765019.271296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765019.271305:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765019.271310:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765019/real 1775765019] req@ffff90c218da2d80 x1862023741818880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765019.271318:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741818880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765019.271322:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765019.271324:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765019.271326:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741818880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765019.271329:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819008:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765019.271338:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765019.271343:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765019/real 1775765019] req@ffff90c218da1880 x1862023741819008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765019.271349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741819008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765019.271355:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765019.271357:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765019.271359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819008:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765020.295023:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765021.319033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765022.343074:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765023.367086:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765024.390993:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765024.391004:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765024.391010:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765024.391022:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 912 00000100:00080000:3.0:1775765024.391035:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765024.391043:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765024.391065:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765024.391084:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741819136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765024.391113:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765024.391117:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765024.391119:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765024.391125:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 912 00000100:00080000:3.0:1775765024.391129:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765024.391135:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765024.391145:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765024.391148:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741819136:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765024.391165:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741819264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765024.391180:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765024.391184:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765024.391186:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765024.391191:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 912 00000100:00080000:3.0:1775765024.391197:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765024.391202:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765024.391209:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765024.391222:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741819392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765024.391233:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765024.391235:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765024.391239:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765024.391241:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765024.391250:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 912 00000100:00080000:3.0:1775765024.391256:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765024.391261:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765024.391268:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765024.391286:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741819520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765024.391297:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765024/real 1775765024] req@ffff90c218da0380 x1862023741819136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765024.391328:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741819136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765024.391342:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765024.391348:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765024.391353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741819136:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765024.391382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765024.391408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765024.391422:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765024/real 1775765024] req@ffff90c218da3480 x1862023741819264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765024.391437:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741819264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765024.391446:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765024.391450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765024.391453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765024.391460:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819392:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765024.391483:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765024.391504:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765024/real 1775765024] req@ffff90c218da0700 x1862023741819392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765024.391517:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741819392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765024.391532:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765024.391536:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765024.391539:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765024.391546:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819520:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765024.391570:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765024.391583:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765024/real 1775765024] req@ffff90c218da0a80 x1862023741819520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765024.391597:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741819520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765024.391619:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765024.391624:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765024.391628:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819520:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765025.415030:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765026.439087:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765027.463036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765028.487027:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765029.510946:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765029.510956:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765029.510961:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765029.510970:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 917 00000100:00080000:3.0:1775765029.510980:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765029.510987:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765029.511013:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765029.511027:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741819648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765029.511051:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765029.511054:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765029.511056:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765029.511059:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 917 00000100:00080000:3.0:1775765029.511062:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765029.511065:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765029.511072:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765029.511082:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741819776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765029.511090:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765029.511094:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741819648:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765029.511095:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765029.511098:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765029.511104:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 917 00000100:00080000:3.0:1775765029.511106:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765029.511109:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765029.511112:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765029.511120:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741819904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765029.511126:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765029.511129:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765029.511130:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765029.511133:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 917 00000100:00080000:3.0:1775765029.511135:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765029.511137:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765029.511139:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765029.511148:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741820032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765029.511188:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765029.511224:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765029.511241:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820032:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765030.535120:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765031.559208:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765032.583197:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765032.583227:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775765032.583274:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765032.583292:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765032.583329:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765032.583344:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765032.583364:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775765032.583388:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765029/real 1775765032] req@ffff90c218da1f80 x1862023741819776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765032.583431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741819776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765032.583448:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765032.583456:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765032.583461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765032.583473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765029/real 1775765032] req@ffff90c218da0000 x1862023741819904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765032.583488:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741819904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765032.583497:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765032.583502:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765032.583506:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741819904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765032.583514:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765029/real 1775765032] req@ffff90c218da0a80 x1862023741820032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765032.583528:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741820032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765032.583541:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765032.583547:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765032.583551:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820032:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775765032.583563:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765029/real 1775765032] req@ffff90c218da3100 x1862023741819648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765032.583576:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741819648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765032.583584:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765032.583588:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765032.583592:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741819648:192.168.202.155@tcp:250:kworker.0 00010000:00010000:0.0:1775765033.607209:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765034.630994:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765034.631002:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765034.631007:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765034.631016:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 923 00000100:00080000:3.0:1775765034.631028:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765034.631035:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765034.631071:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741820160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765034.631100:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765034.631105:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765034.631107:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765034.631111:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 923 00000100:00080000:3.0:1775765034.631114:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765034.631117:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765034.631128:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765034.631139:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741820288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765034.631158:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765034.631160:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741820160:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765034.631168:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765034.631172:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765034.631180:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 923 00000100:00080000:3.0:1775765034.631186:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765034.631192:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765034.631199:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765034.631220:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741820416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765034.631234:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765034.631239:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765034.631242:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775765034.631258:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765034.631259:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 923 00000100:00080000:3.0:1775765034.631267:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765034.631272:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765034.631278:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00010000:00010000:1.0:1775765034.631292:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765034.631295:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741820544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765034.631318:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765034/real 1775765034] req@ffff90c218da0700 x1862023741820160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765034.631349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741820160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765034.631365:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765034.631371:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765034.631376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741820160:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765034.631413:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765034.631447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765034.631463:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765034/real 1775765034] req@ffff90c218da3480 x1862023741820288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765034.631479:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741820288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765034.631489:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765034.631494:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765034.631498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765034.631506:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765034.631542:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765034.631557:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765034/real 1775765034] req@ffff90c218da0380 x1862023741820416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765034.631574:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741820416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765034.631585:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765034.631589:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765034.631594:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820416:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765034.631603:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820544:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765034.631631:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765034.631644:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765034/real 1775765034] req@ffff90c218da1880 x1862023741820544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765034.631661:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741820544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765034.631690:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765034.631695:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765034.631698:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820544:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765035.655237:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765036.679060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765037.703055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765038.727090:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765039.750946:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765039.750955:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765039.750959:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765039.750966:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 928 00000100:00080000:3.0:1775765039.750977:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765039.750984:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765039.751017:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741820672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765039.751036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765039.751046:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765039.751050:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765039.751052:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765039.751055:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 928 00000100:00080000:3.0:1775765039.751057:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765039.751059:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765039.751066:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765039.751076:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741820800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765039.751083:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765039.751086:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765039.751087:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765039.751090:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 928 00000100:00080000:3.0:1775765039.751092:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765039.751094:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765039.751098:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765039.751102:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741820672:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765039.751108:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741820928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765039.751115:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765039.751118:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765039.751119:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765039.751122:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 928 00000100:00080000:3.0:1775765039.751125:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765039.751127:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765039.751129:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765039.751138:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741821056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765039.751162:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765039.751192:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765039/real 1775765039] req@ffff90c218da2d80 x1862023741820672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765039.751205:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741820672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765039.751214:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765039.751216:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765039.751220:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741820672:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765039.751224:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765039.751237:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765039.751244:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765039/real 1775765039] req@ffff90c218da3800 x1862023741820800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765039.751252:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741820800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765039.751256:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765039.751258:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765039.751260:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765039.751278:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820928:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765039.751288:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765039.751294:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765039/real 1775765039] req@ffff90c218da0e00 x1862023741820928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765039.751305:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741820928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765039.751310:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765039.751312:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765039.751314:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741820928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765039.751317:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821056:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765039.751326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765039.751331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765039/real 1775765039] req@ffff90c218da1880 x1862023741821056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765064 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765039.751337:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741821056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765064 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765039.751344:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765039.751345:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765039.751347:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821056:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765040.775100:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765041.799048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765042.823089:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765043.846997:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765044.870953:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765044.870961:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765044.870965:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765044.870973:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 933 00000100:00080000:3.0:1775765044.870983:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765044.870989:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765044.871015:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765044.871028:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741821184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765044.871055:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765044.871058:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765044.871059:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765044.871062:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 933 00000100:00080000:3.0:1775765044.871064:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765044.871067:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765044.871075:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765044.871084:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741821312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765044.871093:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775765044.871097:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741821184:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765044.871097:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765044.871099:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765044.871107:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 933 00000100:00080000:3.0:1775765044.871109:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765044.871111:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765044.871114:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765044.871124:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741821440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765044.871131:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765044.871134:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765044.871135:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765044.871138:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 933 00000100:00080000:3.0:1775765044.871141:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765044.871143:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765044.871146:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765044.871153:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741821568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765044.871160:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765044.871196:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765044/real 1775765044] req@ffff90c218da0380 x1862023741821184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765044.871210:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741821184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765044.871218:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765044.871221:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765044.871224:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741821184:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765044.871240:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765044.871255:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765044.871262:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765044/real 1775765044] req@ffff90c218da3480 x1862023741821312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765044.871270:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741821312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765044.871275:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765044.871278:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765044.871280:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765044.871283:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765044.871315:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765044.871326:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765044/real 1775765044] req@ffff90c218da0700 x1862023741821440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765044.871340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741821440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765044.871350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765044.871354:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765044.871358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765044.871366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821568:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765044.871390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765044.871400:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765044/real 1775765044] req@ffff90c218da3100 x1862023741821568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765044.871410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741821568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765069 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765044.871418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765044.871420:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765044.871423:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821568:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765045.895067:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765046.919025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765047.943079:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765048.967012:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765049.991016:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765049.991026:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765049.991031:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765049.991043:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 938 00000100:00080000:0.0:1775765049.991056:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765049.991065:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765049.991085:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765049.991111:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741821696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765049.991137:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765049.991140:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765049.991141:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765049.991144:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 938 00000100:00080000:0.0:1775765049.991148:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765049.991151:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765049.991161:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765049.991171:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741821824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765049.991180:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741821696:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765049.991181:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765049.991187:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765049.991190:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765049.991194:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 938 00000100:00080000:0.0:1775765049.991196:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765049.991199:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765049.991201:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765049.991209:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741821952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765049.991215:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765049.991218:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765049.991219:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765049.991222:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 938 00000100:00080000:0.0:1775765049.991225:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765049.991227:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765049.991230:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765049.991237:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741822080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765049.991272:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765049.991321:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765049/real 1775765049] req@ffff90c20363bb80 x1862023741821696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765049.991343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741821696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765049.991355:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765049.991360:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765049.991365:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741821696:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765049.991391:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765049.991419:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765049.991433:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765049/real 1775765049] req@ffff90c20363a680 x1862023741821824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765049.991445:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741821824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765049.991454:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765049.991457:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765049.991461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765049.991467:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765049.991484:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765049.991493:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765049/real 1775765049] req@ffff90c20363aa00 x1862023741821952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765049.991504:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741821952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765049.991512:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765049.991515:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765049.991519:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741821952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765049.991525:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822080:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765049.991541:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765049.991549:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765049/real 1775765049] req@ffff90c203638380 x1862023741822080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765049.991578:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741822080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765074 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765049.991589:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765049.991592:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765049.991595:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822080:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765051.015160:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765052.039088:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765053.063000:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765054.087054:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765055.111068:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765055.111082:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765055.111090:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765055.111107:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 943 00000100:00080000:0.0:1775765055.111136:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765055.111149:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765055.111174:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765055.111215:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741822208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765055.111257:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765055.111262:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765055.111265:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765055.111271:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 943 00000100:00080000:0.0:1775765055.111278:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765055.111284:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765055.111308:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765055.111338:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741822336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765055.111338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741822208:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765055.111377:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765055.111382:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765055.111384:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765055.111396:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 943 00000100:00080000:0.0:1775765055.111400:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765055.111405:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765055.111412:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765055.111440:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741822464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765055.111454:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765055.111460:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765055.111462:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765055.111467:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 943 00000100:00080000:0.0:1775765055.111471:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775765055.111475:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765055.111482:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765055.111491:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765055.111510:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741822592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765055.111567:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765055/real 1775765055] req@ffff90c203638e00 x1862023741822208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765055.111618:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741822208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765055.111638:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765055.111696:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765055.111701:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741822208:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765055.111757:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822336:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765055.111798:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765055.111819:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765055/real 1775765055] req@ffff90c20363b480 x1862023741822336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765055.111855:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741822336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765055.111866:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765055.111870:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765055.111873:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765055.111880:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822464:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765055.111959:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765055.111977:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765055/real 1775765055] req@ffff90c20363b800 x1862023741822464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765055.111995:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741822464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765055.112006:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765055.112011:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765055.112016:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765055.112026:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822592:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765055.112059:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765055.112073:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765055/real 1775765055] req@ffff90c203639f80 x1862023741822592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765055.112084:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741822592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765055.112095:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765055.112098:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765055.112102:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822592:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765056.135059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765057.159039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765058.183040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765059.207072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765060.231006:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765060.231015:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765060.231019:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765060.231028:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 948 00000100:00080000:3.0:1775765060.231039:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775765060.231040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765060.231046:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765060.231080:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741822720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765060.231106:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765060.231109:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765060.231111:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765060.231113:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 948 00000100:00080000:3.0:1775765060.231115:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765060.231118:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765060.231125:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765060.231133:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741822848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765060.231141:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765060.231144:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765060.231145:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765060.231147:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 948 00000100:00080000:3.0:1775765060.231150:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765060.231152:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741822720:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765060.231161:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765060.231165:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765060.231174:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741822976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765060.231180:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765060.231183:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765060.231184:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765060.231187:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 948 00000100:00080000:3.0:1775765060.231189:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765060.231191:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765060.231195:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765060.231201:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741823104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765060.231221:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765060.231255:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765060/real 1775765060] req@ffff90c218da0a80 x1862023741822720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765060.231269:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741822720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765060.231279:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765060.231284:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765060.231288:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741822720:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765060.231296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765060.231320:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765060.231333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765060/real 1775765060] req@ffff90c218da0000 x1862023741822848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765060.231343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741822848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765060.231350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765060.231352:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765060.231354:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765060.231381:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765060.231399:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765060.231407:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765060/real 1775765060] req@ffff90c218da1f80 x1862023741822976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765060.231415:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741822976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765060.231420:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765060.231422:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765060.231424:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741822976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765060.231428:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823104:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765060.231437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765060.231442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765060/real 1775765060] req@ffff90c218da3100 x1862023741823104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765060.231449:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741823104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765060.231456:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765060.231457:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765060.231459:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823104:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765061.255088:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765062.279071:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765063.303076:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765064.327044:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765065.351012:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765065.351023:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765065.351028:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765065.351039:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 953 00000100:00080000:0.0:1775765065.351058:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765065.351067:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765065.351069:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765065.351107:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741823232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765065.351135:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765065.351139:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765065.351141:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765065.351146:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 953 00000100:00080000:0.0:1775765065.351150:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765065.351154:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765065.351163:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741823232:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775765065.351164:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765065.351182:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741823360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765065.351194:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765065.351199:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765065.351201:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765065.351204:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 953 00000100:00080000:0.0:1775765065.351209:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765065.351213:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765065.351218:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765065.351220:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765065.351234:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741823488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765065.351247:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765065.351251:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765065.351253:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765065.351258:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 953 00000100:00080000:0.0:1775765065.351263:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765065.351267:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765065/real 1775765065] req@ffff90c203639f80 x1862023741823232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765065.351269:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765065.351278:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765065.351288:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741823232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765065.351294:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741823616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765065.351298:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765065.351307:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765065.351312:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741823232:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765065.351335:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765065.351359:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765065.351372:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765065/real 1775765065] req@ffff90c20363b800 x1862023741823360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765065.351386:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741823360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765065.351394:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765065.351396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765065.351399:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765065.351403:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823488:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765065.351420:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765065.351431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765065/real 1775765065] req@ffff90c20363b480 x1862023741823488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765065.351444:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741823488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765065.351459:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765065.351463:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765065.351467:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765065.351473:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823616:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765065.351489:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765065.351497:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765065/real 1775765065] req@ffff90c203638e00 x1862023741823616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765065.351505:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741823616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765065.351512:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765065.351514:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765065.351516:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823616:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765066.375076:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765067.399058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765068.423072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765069.447072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765070.470967:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765070.470978:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765070.470984:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765070.470996:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 958 00000100:00080000:3.0:1775765070.471009:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765070.471018:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765070.471053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765070.471065:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741823744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765070.471095:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765070.471099:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765070.471101:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765070.471106:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 958 00000100:00080000:3.0:1775765070.471112:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765070.471116:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765070.471127:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741823744:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775765070.471127:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765070.471148:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741823872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765070.471162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765070.471166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765070.471169:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765070.471174:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 958 00000100:00080000:3.0:1775765070.471179:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765070.471182:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765070.471188:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765070.471198:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765070.471205:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741824000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765070.471216:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765070.471220:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765070.471222:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765070.471227:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 958 00000100:00080000:3.0:1775765070.471232:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775765070.471234:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765070/real 1775765070] req@ffff90c218da0700 x1862023741823744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765070.471241:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765070.471250:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765070.471254:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741823744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765070.471268:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775765070.471269:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741824128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765070.471274:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765070.471280:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741823744:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765070.471304:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823872:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765070.471320:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765070.471327:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765070/real 1775765070] req@ffff90c218da3480 x1862023741823872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765070.471335:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741823872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765070.471340:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765070.471342:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765070.471344:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741823872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765070.471347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765070.471357:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765070.471362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765070/real 1775765070] req@ffff90c218da0380 x1862023741824000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765070.471368:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741824000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765070.471373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765070.471375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765070.471376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765070.471381:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824128:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765070.471390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765070.471395:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765070/real 1775765070] req@ffff90c218da1880 x1862023741824128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765070.471402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741824128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765070.471408:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765070.471410:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765070.471412:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824128:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765071.495057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765072.519075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765073.543058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765074.567137:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765075.590979:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765075.590989:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765075.590994:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765075.591004:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 964 00000100:00080000:3.0:1775765075.591018:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765075.591026:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765075.591075:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741824256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765075.591103:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765075.591107:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765075.591109:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765075.591113:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 964 00010000:00010000:0.0:1775765075.591114:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765075.591117:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765075.591121:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765075.591148:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765075.591168:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741824384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765075.591184:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765075.591191:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775765075.591193:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741824256:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765075.591195:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765075.591208:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 964 00000100:00080000:3.0:1775765075.591215:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765075.591221:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765075.591229:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765075.591276:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741824512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765075.591287:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765075.591299:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765075.591305:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765075.591308:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765075.591315:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 964 00000100:00080000:3.0:1775765075.591322:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765075.591327:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765075.591333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765075/real 1775765075] req@ffff90c218da0e00 x1862023741824256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775765075.591335:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765075.591358:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741824256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765075.591363:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741824640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765075.591379:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765075.591389:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765075.591394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741824256:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765075.591420:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765075.591439:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765075.591448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765075/real 1775765075] req@ffff90c218da3800 x1862023741824384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765075.591480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741824384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765075.591488:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765075.591497:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765075.591500:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765075.591506:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765075.591525:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765075.591533:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765075/real 1775765075] req@ffff90c218da2d80 x1862023741824512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765075.591543:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741824512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765075.591549:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765075.591552:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765075.591554:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765075.591559:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824640:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765075.591572:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765075.591578:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765075/real 1775765075] req@ffff90c218da1880 x1862023741824640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765075.591594:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741824640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765075.591602:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765075.591604:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765075.591606:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824640:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765076.615075:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765077.639093:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765078.663054:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765079.687053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765080.710949:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765080.710960:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765080.710966:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765080.710977:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 969 00000100:00080000:3.0:1775765080.710990:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765080.710997:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765080.711042:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741824768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765080.711052:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765080.711063:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765080.711067:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765080.711068:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765080.711072:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 969 00000100:00080000:3.0:1775765080.711075:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765080.711077:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765080.711087:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765080.711088:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741824768:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765080.711098:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741824896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765080.711107:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765080.711110:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765080.711111:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765080.711114:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 969 00000100:00080000:3.0:1775765080.711116:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765080.711118:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765080.711121:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765080.711130:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741825024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765080.711137:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765080.711140:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765080.711141:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765080.711144:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 969 00000100:00080000:3.0:1775765080.711146:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765080.711148:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765080.711151:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765080.711160:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741825152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765080.711168:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765080.711212:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765080/real 1775765080] req@ffff90c218da0380 x1862023741824768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765080.711229:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741824768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765080.711237:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765080.711240:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765080.711243:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741824768:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765080.711265:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824896:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765080.711279:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765080.711287:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765080/real 1775765080] req@ffff90c218da3480 x1862023741824896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765080.711295:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741824896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765080.711301:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765080.711304:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765080.711306:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741824896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765080.711309:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825024:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765080.711319:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765080.711324:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765080/real 1775765080] req@ffff90c218da0700 x1862023741825024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765080.711331:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741825024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765080.711336:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765080.711338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765080.711339:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765080.711343:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825152:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765080.711353:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765080.711358:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765080/real 1775765080] req@ffff90c218da3100 x1862023741825152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765105 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765080.711364:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741825152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765105 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765080.711371:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765080.711373:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765080.711376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825152:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765081.735066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765082.759127:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765083.783068:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765084.807172:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765085.830969:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765085.830977:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765085.830981:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765085.830989:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 974 00000100:00080000:0.0:1775765085.830998:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765085.831004:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765085.831047:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741825280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765085.831050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765085.831073:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765085.831076:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765085.831078:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765085.831081:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 974 00000100:00080000:0.0:1775765085.831084:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765085.831087:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765085.831094:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765085.831101:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741825280:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765085.831106:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741825408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765085.831115:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765085.831117:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765085.831118:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765085.831120:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 974 00000100:00080000:0.0:1775765085.831122:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765085.831124:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765085.831127:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765085.831134:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741825536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765085.831141:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765085.831144:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765085.831145:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765085.831148:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 974 00000100:00080000:0.0:1775765085.831157:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765085.831160:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765085.831164:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:1.0:1775765085.831172:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765085.831173:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741825664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765085.831214:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765085/real 1775765085] req@ffff90c203638380 x1862023741825280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765085.831234:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741825280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765085.831245:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765085.831249:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765085.831253:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741825280:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765085.831275:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765085.831296:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765085.831308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765085/real 1775765085] req@ffff90c20363aa00 x1862023741825408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765085.831320:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741825408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765085.831329:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765085.831333:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765085.831336:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765085.831342:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825536:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765085.831379:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765085.831391:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765085/real 1775765085] req@ffff90c20363a680 x1862023741825536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765085.831403:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741825536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765085.831412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765085.831416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765085.831419:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765085.831426:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825664:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765085.831445:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765085.831456:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765085/real 1775765085] req@ffff90c20363bb80 x1862023741825664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765085.831467:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741825664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765110 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765085.831478:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765085.831481:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765085.831485:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825664:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765086.855059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765087.879030:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765088.903067:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765089.927030:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765090.950962:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765090.950970:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765090.950974:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765090.950983:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 979 00000100:00080000:0.0:1775765090.951001:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765090.951007:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765090.951024:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765090.951043:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741825792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765090.951064:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765090.951067:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765090.951069:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765090.951071:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 979 00000100:00080000:0.0:1775765090.951074:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765090.951076:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765090.951083:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765090.951093:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741825920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765090.951106:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765090.951112:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775765090.951113:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741825792:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765090.951114:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765090.951124:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 979 00000100:00080000:0.0:1775765090.951128:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765090.951131:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765090.951137:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765090.951153:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741826048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765090.951164:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775765090.951174:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765090.951177:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765090.951179:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765090.951183:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 979 00000100:00080000:0.0:1775765090.951187:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765090.951190:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765090.951195:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765090.951210:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741826176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765090.951212:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765090/real 1775765090] req@ffff90c20363b100 x1862023741825792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765090.951252:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741825792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765090.951259:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765090.951263:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765090.951266:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741825792:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765090.951283:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765090.951297:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765090.951986:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765090/real 1775765090] req@ffff90c203639180 x1862023741825920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765090.951999:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741825920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765090.952007:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765090.952010:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765090.952014:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741825920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765090.952020:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826048:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765090.952039:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765090.952050:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765090/real 1775765090] req@ffff90c203638700 x1862023741826048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765090.952061:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741826048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765090.952075:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765090.952078:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765090.952082:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765090.952087:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826176:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765090.952100:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765090.952108:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765090/real 1775765090] req@ffff90c203638000 x1862023741826176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765090.952118:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741826176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765115 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765090.952128:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765090.952131:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765090.952134:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826176:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765091.975001:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765092.999086:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765094.023037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765095.047128:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765096.071014:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765096.071028:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765096.071038:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765096.071043:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765096.071054:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 984 00000100:00080000:1.0:1775765096.071067:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765096.071073:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765096.071119:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741826304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765096.071163:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765096.071166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765096.071168:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765096.071171:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 984 00000100:00080000:1.0:1775765096.071173:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765096.071177:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765096.071187:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765096.071198:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741826432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765096.071208:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765096.071210:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741826304:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765096.071212:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765096.071220:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765096.071234:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 984 00000100:00080000:1.0:1775765096.071237:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765096.071239:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765096.071243:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765096.071253:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741826560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765096.071259:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765096.071278:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765096.071280:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765096.071285:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 984 00000100:00080000:1.0:1775765096.071299:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765096.071303:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765096.071308:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765096.071318:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741826688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765096.071364:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765096.071404:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765096.071435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826688:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765097.095100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765098.119063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765099.143109:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775765099.143319:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775765099.143386:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765099.143410:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775765099.143463:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775765099.143476:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775765099.143492:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775765099.143513:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765096/real 1775765099] req@ffff90c203494e00 x1862023741826432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765099.143550:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741826432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765099.143566:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765099.143574:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765099.143579:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765099.143590:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765096/real 1775765099] req@ffff90c203497480 x1862023741826560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765099.143613:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741826560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765099.143622:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765099.143627:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765099.143631:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765099.143639:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765096/real 1775765099] req@ffff90c203494000 x1862023741826688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765099.143652:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741826688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765099.143666:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765099.143670:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765099.143675:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826688:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775765099.143683:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765096/real 1775765099] req@ffff90c203495f80 x1862023741826304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765099.143696:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741826304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765099.143704:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765099.143708:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765099.143711:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741826304:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775765100.167153:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765101.190990:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765101.191000:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765101.191005:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765101.191016:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 989 00000100:00080000:3.0:1775765101.191029:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765101.191037:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765101.191084:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741826816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765101.191102:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765101.191110:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765101.191115:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765101.191118:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765101.191123:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 989 00000100:00080000:3.0:1775765101.191128:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765101.191132:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765101.191145:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765101.191148:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741826816:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765101.191163:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741826944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765101.191190:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765101.191195:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765101.191197:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765101.191202:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 989 00000100:00080000:3.0:1775765101.191207:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765101.191242:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765101.191251:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765101.191253:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765101.191273:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741827072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765101.191285:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765101.191289:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765101.191291:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765101.191296:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 989 00000100:00080000:3.0:1775765101.191302:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765101.191308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765101/real 1775765101] req@ffff90c218da1f80 x1862023741826816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765101.191308:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765101.191318:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765101.191337:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741827200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765101.191340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741826816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765101.191385:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765101.191390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765101.191395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741826816:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765101.191434:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765101.191467:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765101.191482:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765101/real 1775765101] req@ffff90c218da0000 x1862023741826944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765101.191497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741826944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765101.191513:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765101.191517:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765101.191520:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741826944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765101.191528:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765101.191557:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765101.191568:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765101/real 1775765101] req@ffff90c218da0a80 x1862023741827072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765101.191587:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741827072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765101.191597:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765101.191601:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765101.191605:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765101.191611:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827200:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765101.191636:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765101.191651:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765101/real 1775765101] req@ffff90c218da3100 x1862023741827200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765101.191672:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741827200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765101.191684:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765101.191687:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765101.191691:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827200:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765102.215263:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765103.239062:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765104.263118:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765105.287135:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765106.310946:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765106.310954:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765106.310958:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765106.310971:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 994 00000100:00080000:3.0:1775765106.310981:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765106.310987:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765106.311007:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765106.311020:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741827328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765106.311040:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765106.311043:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765106.311044:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765106.311047:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 994 00000100:00080000:3.0:1775765106.311049:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765106.311053:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765106.311061:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765106.311071:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741827328:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765106.311072:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741827456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765106.311084:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765106.311086:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765106.311088:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765106.311091:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 994 00000100:00080000:3.0:1775765106.311093:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765106.311095:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765106.311098:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765106.311107:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741827584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765106.311113:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765106.311115:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765106.311117:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765106.311120:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 994 00000100:00080000:3.0:1775765106.311123:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765106.311125:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765106.311128:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775765106.311130:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765106.311139:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741827712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765106.311182:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765106/real 1775765106] req@ffff90c218da0700 x1862023741827328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765106.311208:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741827328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765106.311215:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765106.311218:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765106.311221:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741827328:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765106.311239:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765106.311255:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765106.311264:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765106/real 1775765106] req@ffff90c218da3480 x1862023741827456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765106.311276:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741827456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765106.311281:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765106.311283:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765106.311285:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765106.311289:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827584:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765106.311299:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765106.311305:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765106/real 1775765106] req@ffff90c218da0380 x1862023741827584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765106.311312:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741827584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765106.311317:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765106.311318:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765106.311320:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765106.311323:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827712:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765106.311332:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765106.311337:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765106/real 1775765106] req@ffff90c218da1880 x1862023741827712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765106.311343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741827712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765106.311354:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765106.311356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765106.311358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827712:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765107.335045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765108.358996:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765109.383040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765110.407017:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765111.430931:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765111.430944:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765111.430949:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765111.430958:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 999 00000100:00080000:3.0:1775765111.430968:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765111.430973:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765111.431006:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741827840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765111.431024:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765111.431032:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765111.431035:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765111.431036:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765111.431040:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 999 00000100:00080000:3.0:1775765111.431046:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765111.431049:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765111.431056:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765111.431064:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741827968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765111.431070:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765111.431073:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765111.431074:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765111.431076:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 999 00000100:00080000:3.0:1775765111.431078:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765111.431080:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765111.431083:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765111.431092:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741828096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765111.431100:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765111.431102:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765111.431104:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765111.431106:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 999 00000100:00080000:3.0:1775765111.431108:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765111.431110:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765111.431114:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765111.431158:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741827840:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765111.431159:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741828224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765111.431253:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765111.431298:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765111/real 1775765111] req@ffff90c218da2d80 x1862023741827840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765111.431315:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741827840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765111.431324:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765111.431327:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765111.431331:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741827840:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765111.431336:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765111.431351:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765111.431359:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765111/real 1775765111] req@ffff90c218da3800 x1862023741827968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765111.431367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741827968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765111.431372:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765111.431374:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765111.431376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741827968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765111.431379:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765111.431390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765111.431395:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765111/real 1775765111] req@ffff90c218da0e00 x1862023741828096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765111.431402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741828096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765111.431407:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765111.431409:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765111.431411:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765111.431435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828224:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765111.431446:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765111.431451:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765111/real 1775765111] req@ffff90c218da1880 x1862023741828224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765111.431463:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741828224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765111.431470:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765111.431472:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765111.431474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828224:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765112.455155:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765113.479041:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765114.503141:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765115.527007:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765116.550974:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765116.550983:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765116.550987:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765116.550995:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1005 00000100:00080000:0.0:1775765116.551005:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765116.551012:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765116.551019:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765116.551048:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741828352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765116.551069:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765116.551089:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765116.551091:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765116.551094:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1005 00000100:00080000:0.0:1775765116.551098:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765116.551102:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765116.551110:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765116.551115:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741828352:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765116.551125:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741828480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765116.551133:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765116.551136:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765116.551138:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765116.551141:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1005 00000100:00080000:0.0:1775765116.551143:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765116.551146:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765116.551149:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765116.551156:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741828608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765116.551162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765116.551164:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765116.551166:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765116.551169:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1005 00000100:00080000:0.0:1775765116.551171:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765116.551173:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765116.551176:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:1.0:1775765116.551181:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765116.551185:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741828736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765116.551222:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765116/real 1775765116] req@ffff90c203639500 x1862023741828352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765116.551238:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741828352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765116.551254:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765116.551257:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765116.551260:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741828352:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765116.551277:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765116.551294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765116.551302:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765116/real 1775765116] req@ffff90c203638000 x1862023741828480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765116.551310:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741828480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765116.551319:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765116.551322:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765116.551324:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765116.551328:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765116.551346:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765116.551357:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765116/real 1775765116] req@ffff90c203638700 x1862023741828608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765116.551369:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741828608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765116.551377:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765116.551380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765116.551383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765116.551389:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828736:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765116.551411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765116.551423:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765116/real 1775765116] req@ffff90c203639180 x1862023741828736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765116.551436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741828736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765116.551447:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765116.551451:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765116.551455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828736:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765117.575077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765118.599042:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765119.623115:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765120.647141:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765121.670987:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765121.670996:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765121.671001:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765121.671012:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1010 00010000:00010000:0.0:1775765121.671014:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765121.671023:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765121.671049:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765121.671087:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741828864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765121.671116:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765121.671119:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765121.671121:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765121.671125:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1010 00000100:00080000:3.0:1775765121.671129:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765121.671132:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765121.671142:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765121.671154:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741828992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765121.671165:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765121.671169:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765121.671170:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765121.671175:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1010 00000100:00080000:3.0:1775765121.671178:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765121.671182:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765121.671188:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765121.671199:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741828864:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765121.671202:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741829120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765121.671224:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765121.671228:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765121.671230:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765121.671235:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1010 00000100:00080000:3.0:1775765121.671238:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765121.671242:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765121.671248:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765121.671262:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741829248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765121.671305:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765121.671353:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765121/real 1775765121] req@ffff90c218da0380 x1862023741828864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765121.671370:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741828864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765121.671380:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765121.671383:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765121.671387:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741828864:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765121.671393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828992:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765121.671408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765121.671416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765121/real 1775765121] req@ffff90c218da3480 x1862023741828992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765121.671424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741828992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765121.671429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765121.671431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765121.671433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741828992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765121.671454:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765121.671467:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765121.671473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765121/real 1775765121] req@ffff90c218da0700 x1862023741829120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765121.671481:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741829120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765121.671486:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765121.671488:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765121.671490:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765121.671494:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829248:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765121.671504:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765121.671509:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765121/real 1775765121] req@ffff90c218da3100 x1862023741829248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765121.671516:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741829248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765121.671523:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765121.671525:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765121.671527:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829248:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765122.695107:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765123.719128:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765124.743087:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765125.767104:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765126.791021:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765126.791035:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765126.791044:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765126.791070:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1015 00000100:00080000:3.0:1775765126.791086:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765126.791097:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765126.791103:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765126.791150:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741829376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765126.791203:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765126.791210:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765126.791213:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765126.791221:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1015 00000100:00080000:3.0:1775765126.791228:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765126.791236:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765126.791260:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765126.791275:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741829376:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765126.791288:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741829504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765126.791318:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765126.791325:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765126.791328:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765126.791337:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1015 00000100:00080000:3.0:1775765126.791344:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775765126.791364:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765126.791371:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765126.791392:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765126.791420:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741829632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765126.791429:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765126/real 1775765126] req@ffff90c218da0a80 x1862023741829376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765126.791473:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741829376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765126.791474:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765126.791483:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765126.791495:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765126.791509:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1015 00000100:00080000:1.0:1775765126.791513:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1775765126.791516:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765126.791522:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00080000:3.0:1775765126.791527:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765126.791532:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741829376:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775765126.791539:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765126.791560:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741829760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765126.791565:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765126.791623:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765126.791637:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765126/real 1775765126] req@ffff90c218da0000 x1862023741829504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765126.791658:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741829504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765126.791667:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765126.791670:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765126.791674:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765126.791680:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765126.791696:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765126.791705:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765126/real 1775765126] req@ffff90c218da1f80 x1862023741829632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765126.791717:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741829632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765126.791733:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765126.791737:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765126.791740:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765126.791753:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829760:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765126.791770:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765126.791778:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765126/real 1775765126] req@ffff90c218da3100 x1862023741829760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765151 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765126.791789:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741829760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765151 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765126.791800:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765126.791803:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765126.791807:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741829760:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765127.815098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765128.839152:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765129.863106:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765130.887082:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765131.911119:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765131.911151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765131.911210:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765131.911215:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765131.911224:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1020 00000100:00080000:0.0:1775765131.911236:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765131.911242:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765131.911278:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741829888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765131.911305:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765131.911308:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765131.911310:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765131.911313:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1020 00000100:00080000:0.0:1775765131.911316:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765131.911319:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765131.911328:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765131.911338:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741830016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765131.911347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741829888:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765131.911366:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765131.911369:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765131.911370:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765131.911374:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1020 00000100:00080000:0.0:1775765131.911377:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765131.911380:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765131.911384:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765131.911394:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741830144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765131.911401:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765131.911404:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765131.911406:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765131.911408:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1020 00000100:00080000:0.0:1775765131.911413:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765131.911416:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765131.911420:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:1.0:1775765131.911429:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765131.911430:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741830272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765131.911472:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765131/real 1775765131] req@ffff90c20363b100 x1862023741829888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765131.911489:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741829888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765131.911499:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765131.911503:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765131.911506:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741829888:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765131.911527:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765131.911546:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765131.911556:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765131/real 1775765131] req@ffff90c20363bb80 x1862023741830016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765131.911567:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741830016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765131.911585:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765131.911590:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765131.911594:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765131.911603:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765131.911630:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765131.911645:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765131/real 1775765131] req@ffff90c20363a680 x1862023741830144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765131.911661:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741830144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765131.911670:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765131.911673:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765131.911676:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765131.911681:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830272:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765131.911700:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765131.911708:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765131/real 1775765131] req@ffff90c20363aa00 x1862023741830272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765131.911717:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741830272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765156 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765131.911743:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765131.911747:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765131.911750:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830272:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765132.935184:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765133.959100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765134.983053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765136.007115:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765137.031087:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765137.031114:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765137.031121:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765137.031134:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1025 00000100:00080000:3.0:1775765137.031150:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765137.031161:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765137.031208:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765137.031217:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741830400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765137.031272:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765137.031279:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765137.031283:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765137.031291:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1025 00000100:00080000:3.0:1775765137.031298:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765137.031306:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765137.031323:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765137.031346:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741830528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765137.031363:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765137.031368:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765137.031371:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765137.031377:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1025 00000100:00080000:3.0:1775765137.031382:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765137.031387:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765137.031395:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765137.031407:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741830656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765137.031427:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775765137.031429:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741830400:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765137.031433:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765137.031437:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765137.031452:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1025 00000100:00080000:3.0:1775765137.031459:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765137.031463:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765137.031469:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765137.031484:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741830784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765137.031587:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765137.031668:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765137/real 1775765137] req@ffff90c218da0700 x1862023741830400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765137.031705:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741830400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765137.031722:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765137.031727:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765137.031733:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741830400:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765137.031746:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830528:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765137.031789:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765137.031813:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765137/real 1775765137] req@ffff90c218da3480 x1862023741830528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765137.031854:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741830528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765137.031979:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765137.031986:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765137.031991:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765137.032034:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765137.032068:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765137.032084:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765137/real 1775765137] req@ffff90c218da0380 x1862023741830656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765137.032099:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741830656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765137.032109:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765137.032114:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765137.032118:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765137.032125:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830784:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765137.032144:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765137.032154:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765137/real 1775765137] req@ffff90c218da1880 x1862023741830784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765137.032168:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741830784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765137.032189:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765137.032194:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765137.032197:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741830784:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765138.055177:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765139.079139:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765140.103083:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765141.127098:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765142.150957:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765142.150964:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765142.150970:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765142.150979:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1030 00000100:00080000:3.0:1775765142.150990:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765142.150995:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765142.151030:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741830912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765142.151054:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765142.151058:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765142.151060:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765142.151064:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1030 00000100:00080000:3.0:1775765142.151066:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765142.151070:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765142.151077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00100000:3.0:1775765142.151079:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765142.151091:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741831040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765142.151102:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765142.151111:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775765142.151112:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741830912:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765142.151113:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765142.151128:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1030 00000100:00080000:3.0:1775765142.151131:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765142.151135:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765142.151137:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765142.151144:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741831168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765142.151156:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765142.151158:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765142.151160:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765142.151163:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1030 00000100:00080000:3.0:1775765142.151165:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765142.151168:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765142.151171:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765142.151179:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741831296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765142.151205:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765142.151247:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765142/real 1775765142] req@ffff90c218da0e00 x1862023741830912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765142.151269:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741830912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765142.151283:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765142.151289:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765142.151294:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741830912:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765142.151326:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765142.151350:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765142.151361:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765142/real 1775765142] req@ffff90c218da3800 x1862023741831040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765142.151371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741831040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765142.151378:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765142.151380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765142.151383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765142.151389:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765142.151402:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765142.151409:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765142/real 1775765142] req@ffff90c218da2d80 x1862023741831168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765142.151418:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741831168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765142.151443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765142.151446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765142.151449:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765142.151455:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831296:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765142.151473:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765142.151482:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765142/real 1775765142] req@ffff90c218da1880 x1862023741831296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765142.151493:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741831296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765142.151503:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765142.151506:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765142.151508:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831296:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765143.175143:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765144.199030:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765145.223049:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765146.247014:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765147.270961:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765147.270970:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765147.270975:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765147.270984:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1035 00000100:00080000:3.0:1775765147.270994:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765147.271001:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765147.271040:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741831424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765147.271056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765147.271064:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765147.271068:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765147.271070:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765147.271077:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1035 00000100:00080000:3.0:1775765147.271080:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765147.271083:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765147.271091:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765147.271101:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741831552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765147.271110:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765147.271114:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765147.271117:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765147.271125:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1035 00000100:00100000:1.0:1775765147.271127:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741831424:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765147.271132:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765147.271138:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765147.271143:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765147.271169:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741831680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765147.271183:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765147.271187:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:1.0:1775765147.271191:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765147.271192:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765147.271198:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1035 00000100:00080000:3.0:1775765147.271203:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765147.271207:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765147.271213:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765147.271226:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741831808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765147.271233:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765147/real 1775765147] req@ffff90c218da0380 x1862023741831424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765147.271254:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741831424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765147.271263:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765147.271266:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765147.271270:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741831424:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765147.271275:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831552:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765147.271291:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765147.271299:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765147/real 1775765147] req@ffff90c218da3480 x1862023741831552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765147.271308:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741831552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765147.271313:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765147.271316:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765147.271318:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765147.271337:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765147.271349:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765147.271355:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765147/real 1775765147] req@ffff90c218da0700 x1862023741831680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765147.271363:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741831680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765147.271374:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765147.271376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765147.271378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765147.271382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831808:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765147.271393:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765147.271399:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765147/real 1775765147] req@ffff90c218da3100 x1862023741831808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765147.271407:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741831808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765147.271414:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765147.271416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765147.271418:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741831808:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765148.294999:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765149.319045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765150.343000:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765151.367098:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765152.390978:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765152.390990:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765152.390998:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765152.391011:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1040 00000100:00080000:3.0:1775765152.391025:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775765152.391032:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765152.391034:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765152.391086:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741831936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765152.391115:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765152.391119:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765152.391122:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765152.391127:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1040 00000100:00080000:3.0:1775765152.391131:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765152.391136:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765152.391150:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765152.391167:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741832064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765152.391172:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741831936:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765152.391180:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765152.391183:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765152.391185:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765152.391190:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1040 00000100:00080000:3.0:1775765152.391195:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765152.391199:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765152.391204:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765152.391214:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741832192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765152.391230:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765152.391234:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765152.391235:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765152.391240:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1040 00000100:00080000:3.0:1775765152.391245:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765152.391249:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765152.391254:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:1.0:1775765152.391259:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765152.391266:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741832320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765152.391312:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765152/real 1775765152] req@ffff90c218da1f80 x1862023741831936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765152.391337:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741831936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765152.391347:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765152.391351:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765152.391355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741831936:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765152.391382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765152.391404:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765152.391414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765152/real 1775765152] req@ffff90c218da0000 x1862023741832064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765152.391424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741832064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765152.391430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765152.391433:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765152.391435:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765152.391439:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765152.391452:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765152.391458:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765152/real 1775765152] req@ffff90c218da0a80 x1862023741832192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765152.391466:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741832192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765152.391472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765152.391474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765152.391476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765152.391480:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832320:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765152.391491:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765152.391497:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765152/real 1775765152] req@ffff90c218da3100 x1862023741832320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765152.391506:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741832320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765152.391514:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765152.391516:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765152.391518:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832320:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765153.414969:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765154.439077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765155.463049:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765156.487046:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765157.510949:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765157.510957:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765157.510961:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765157.510969:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1045 00000100:00080000:3.0:1775765157.510978:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765157.510984:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765157.511003:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765157.511018:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741832448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765157.511040:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765157.511043:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765157.511044:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765157.511047:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1045 00000100:00080000:3.0:1775765157.511049:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765157.511052:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765157.511060:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765157.511075:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741832576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765157.511084:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765157.511086:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765157.511088:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775765157.511093:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741832448:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765157.511094:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1045 00000100:00080000:3.0:1775765157.511100:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765157.511103:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765157.511107:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765157.511115:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741832704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765157.511122:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765157.511125:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765157.511126:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765157.511130:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1045 00000100:00080000:3.0:1775765157.511132:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765157.511134:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765157.511138:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765157.511147:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741832832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765157.511152:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765157.511185:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765157/real 1775765157] req@ffff90c218da0700 x1862023741832448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765157.511200:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741832448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765157.511211:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765157.511215:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765157.511220:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741832448:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765157.511244:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765157.511269:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765157.511282:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765157/real 1775765157] req@ffff90c218da3480 x1862023741832576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765157.511291:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741832576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765157.511302:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765157.511304:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765157.511306:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765157.511310:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765157.511324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765157.511331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765157/real 1775765157] req@ffff90c218da0380 x1862023741832704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765157.511338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741832704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765157.511343:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765157.511345:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765157.511347:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765157.511351:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832832:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765157.511361:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765157.511369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765157/real 1775765157] req@ffff90c218da1880 x1862023741832832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765157.511394:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741832832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765157.511405:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765157.511409:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765157.511413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741832832:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765158.535051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765159.559101:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765160.583091:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765161.607092:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765162.630969:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765162.630978:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765162.630983:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765162.630992:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1051 00000100:00080000:3.0:1775765162.631003:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765162.631010:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765162.631034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765162.631046:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741832960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765162.631076:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765162.631080:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765162.631081:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765162.631084:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1051 00000100:00080000:3.0:1775765162.631087:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765162.631089:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765162.631097:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765162.631105:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741833088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765162.631113:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765162.631115:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765162.631116:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765162.631136:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1051 00000100:00080000:3.0:1775765162.631139:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765162.631143:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765162.631157:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765162.631166:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741833216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765162.631182:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765162.631189:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1775765162.631190:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741832960:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765162.631191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765162.631200:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1051 00000100:00080000:3.0:1775765162.631210:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765162.631215:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765162.631222:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765162.631242:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741833344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765162.631298:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765162.631338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765162.631383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833344:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765163.655145:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765164.679106:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765165.703030:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765165.703092:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775765165.703127:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765165.703141:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765165.703173:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765165.703181:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765165.703190:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:1.0:1775765165.703234:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765162/real 1775765165] req@ffff90c218da0e00 x1862023741833216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765165.703261:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741833216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765165.703273:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765165.703278:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765165.703282:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765165.703290:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765162/real 1775765165] req@ffff90c218da1880 x1862023741833344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765165.703304:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741833344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765165.703314:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765165.703318:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765165.703321:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833344:192.168.202.155@tcp:8:kworker.0 00000100:00100000:1.0:1775765165.703328:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765162/real 1775765165] req@ffff90c218da2d80 x1862023741832960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765165.703342:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741832960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765165.703350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765165.703353:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765165.703356:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741832960:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765165.703364:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765162/real 1775765165] req@ffff90c218da3800 x1862023741833088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765165.703376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741833088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765165.703383:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765165.703386:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765165.703390:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833088:192.168.202.155@tcp:38:kworker.0 00010000:00010000:0.0:1775765166.727093:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765167.751014:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765167.751026:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765167.751030:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765167.751039:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1056 00000100:00080000:1.0:1775765167.751049:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765167.751054:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765167.751090:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741833472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765167.751100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765167.751119:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765167.751127:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765167.751129:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765167.751133:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1056 00000100:00080000:1.0:1775765167.751137:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765167.751141:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765167.751150:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765167.751158:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741833472:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765167.751163:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741833600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765167.751173:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765167.751176:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765167.751177:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765167.751181:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1056 00000100:00080000:1.0:1775765167.751184:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765167.751187:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765167.751191:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765167.751200:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741833728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765167.751207:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765167.751210:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765167.751211:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765167.751215:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1056 00000800:00000100:2.0:1775765167.751216:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775765167.751218:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765167.751221:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765167.751226:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765167.751235:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741833856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765167.751260:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765167/real 1775765167] req@ffff90c203497100 x1862023741833472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765167.751279:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741833472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765167.751289:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765167.751293:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765167.751298:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741833472:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765167.751327:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765167.751353:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765167.751366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765167/real 1775765167] req@ffff90c203496d80 x1862023741833600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765167.751387:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741833600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765167.751396:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765167.751400:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765167.751403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765167.751408:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765167.751428:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765167.751449:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765167/real 1775765167] req@ffff90c203494380 x1862023741833728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765167.751459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741833728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765167.751467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765167.751471:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765167.751474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765167.751480:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833856:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765167.751502:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765167.751513:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765167/real 1775765167] req@ffff90c203495c00 x1862023741833856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765192 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765167.751527:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741833856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765192 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765167.751539:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765167.751542:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765167.751545:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741833856:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775765168.775047:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765169.799110:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765170.823045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765171.846993:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765172.870993:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765172.871004:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765172.871010:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765172.871023:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1061 00000100:00080000:0.0:1775765172.871036:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765172.871044:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765172.871087:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765172.871092:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741833984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765172.871121:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765172.871124:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765172.871126:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765172.871129:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1061 00000100:00080000:0.0:1775765172.871131:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765172.871134:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765172.871144:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765172.871156:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741834112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765172.871166:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765172.871170:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765172.871172:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765172.871174:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1061 00000100:00080000:0.0:1775765172.871177:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765172.871182:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765172.871189:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765172.871192:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741833984:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765172.871200:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741834240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765172.871216:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765172.871218:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765172.871219:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765172.871223:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1061 00000100:00080000:0.0:1775765172.871224:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765172.871226:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765172.871229:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765172.871240:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741834368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765172.871294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765172.871334:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765172/real 1775765172] req@ffff90c203638380 x1862023741833984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765172.871352:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741833984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765172.871363:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765172.871367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765172.871371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741833984:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765172.871394:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765172.871411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765172.871421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765172/real 1775765172] req@ffff90c203638e00 x1862023741834112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765172.871431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741834112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765172.871438:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765172.871441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765172.871444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765172.871449:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834240:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765172.871461:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765172.871468:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765172/real 1775765172] req@ffff90c20363b480 x1862023741834240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765172.871477:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741834240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765172.871484:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765172.871486:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765172.871489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765172.871493:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834368:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765172.871505:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765172.871513:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765172/real 1775765172] req@ffff90c20363b800 x1862023741834368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765172.871529:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741834368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765197 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765172.871590:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765172.871597:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765172.871621:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834368:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765173.895071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765174.918979:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765175.943028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765176.966991:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765177.991024:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765177.991035:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765177.991042:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765177.991055:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1066 00010000:00010000:2.0:1775765177.991055:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765177.991072:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765177.991082:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765177.991155:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741834496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765177.991191:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765177.991195:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765177.991197:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765177.991201:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1066 00000100:00080000:0.0:1775765177.991206:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765177.991211:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765177.991224:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765177.991240:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741834624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765177.991253:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741834496:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765177.991257:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765177.991275:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765177.991278:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765177.991284:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1066 00000100:00080000:0.0:1775765177.991287:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765177.991290:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765177.991294:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765177.991307:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741834752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765177.991315:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765177.991318:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765177.991320:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765177.991325:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1066 00000100:00080000:0.0:1775765177.991329:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765177.991332:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765177.991338:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765177.991347:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741834880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765177.991368:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765177.991421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765177/real 1775765177] req@ffff90c203639f80 x1862023741834496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765177.991439:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741834496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765177.991449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765177.991453:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765177.991457:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741834496:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765177.991481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765177.991558:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765177.991575:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765177/real 1775765177] req@ffff90c20363b800 x1862023741834624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765177.991592:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741834624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765177.991601:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765177.991606:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765177.991609:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765177.991617:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765177.991637:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765177.991646:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765177/real 1775765177] req@ffff90c20363b480 x1862023741834752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765177.991655:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741834752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765177.991661:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765177.991664:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765177.991666:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765177.991671:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834880:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765177.991682:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765177.991688:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765177/real 1775765177] req@ffff90c203638e00 x1862023741834880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765177.991696:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741834880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765202 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765177.991704:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765177.991706:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765177.991708:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741834880:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765179.015194:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765180.039041:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765181.063002:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765182.087029:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765183.110959:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765183.110970:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765183.110976:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765183.110987:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1071 00000100:00080000:0.0:1775765183.110999:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765183.111008:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765183.111048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765183.111059:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741835008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765183.111087:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765183.111090:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765183.111091:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765183.111095:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1071 00000100:00080000:0.0:1775765183.111097:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765183.111100:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765183.111111:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765183.111121:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741835136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765183.111134:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741835008:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765183.111136:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765183.111150:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765183.111152:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765183.111158:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1071 00000100:00080000:0.0:1775765183.111159:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765183.111162:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765183.111166:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765183.111176:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741835264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765183.111182:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765183.111184:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765183.111186:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765183.111189:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1071 00000100:00080000:0.0:1775765183.111191:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765183.111193:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765183.111196:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765183.111205:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741835392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765183.111216:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765183.111255:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765183/real 1775765183] req@ffff90c203638380 x1862023741835008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765183.111274:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741835008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765183.111283:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765183.111286:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765183.111290:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741835008:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765183.111310:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765183.111332:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765183.111341:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765183/real 1775765183] req@ffff90c20363aa00 x1862023741835136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765183.111349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741835136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765183.111354:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765183.111356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765183.111358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765183.111362:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765183.111373:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765183.111378:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765183/real 1775765183] req@ffff90c20363a680 x1862023741835264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765183.111385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741835264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765183.111394:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765183.111397:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765183.111399:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765183.111402:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835392:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765183.111412:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765183.111417:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765183/real 1775765183] req@ffff90c20363bb80 x1862023741835392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765183.111424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741835392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765183.111431:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765183.111433:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765183.111434:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835392:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775765184.135043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765185.159101:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765186.183134:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765187.207011:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765188.231010:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765188.231023:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765188.231041:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765188.231055:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1076 00000100:00080000:0.0:1775765188.231070:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765188.231080:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765188.231136:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741835520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765188.231168:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765188.231191:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765188.231198:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765188.231201:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765188.231208:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1076 00000100:00080000:0.0:1775765188.231221:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765188.231229:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765188.231245:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765188.231252:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741835520:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765188.231274:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741835648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765188.231294:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765188.231300:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765188.231302:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765188.231307:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1076 00000100:00080000:0.0:1775765188.231314:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765188.231318:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765188.231324:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765188.231338:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741835776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765188.231349:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765188.231353:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765188.231355:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765188.231362:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765188.231368:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1076 00000100:00080000:0.0:1775765188.231373:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765188.231377:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765188.231384:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765188.231397:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741835904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765188.231437:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765188/real 1775765188] req@ffff90c20363b100 x1862023741835520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765188.231462:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741835520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765188.231475:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765188.231481:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765188.231486:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741835520:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765188.231516:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835648:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765188.231540:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765188.231553:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765188/real 1775765188] req@ffff90c203639180 x1862023741835648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765188.231566:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741835648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765188.231574:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765188.231577:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765188.231581:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765188.231586:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835776:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765188.231601:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765188.231610:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765188/real 1775765188] req@ffff90c203638700 x1862023741835776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765188.231620:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741835776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765188.231629:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765188.231632:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765188.231634:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765188.231640:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835904:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765188.231656:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765188.231663:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765188/real 1775765188] req@ffff90c203638000 x1862023741835904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765188.231674:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741835904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765188.231685:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765188.231688:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765188.231691:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741835904:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765189.255085:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765190.279008:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765191.302994:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765192.327135:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765193.351071:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765193.351085:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765193.351092:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765193.351104:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1081 00000100:00080000:2.0:1775765193.351141:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765193.351152:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765193.351218:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be680 x1862023741836032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765193.351262:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765193.351268:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765193.351270:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765193.351276:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1081 00000100:00080000:2.0:1775765193.351280:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765193.351285:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765193.351312:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765193.351327:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bed80 x1862023741836160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765193.351341:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765193.351345:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765193.351348:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765193.351352:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1081 00000100:00080000:2.0:1775765193.351356:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765193.351403:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765193.351428:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765193.351442:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023741836288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765193.351454:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741836032:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775765193.351457:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765193.351470:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765193.351472:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765193.351478:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1081 00000100:00080000:2.0:1775765193.351484:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765193.351488:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765193.351497:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765193.351511:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023741836416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775765193.351579:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00010000:00010000:2.0:1775765193.351603:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765193.351637:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765193/real 1775765193] req@ffff90c2034be680 x1862023741836032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765193.351660:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be680 x1862023741836032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765193.351672:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765193.351705:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765193.351710:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741836032:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765193.351721:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765193.351770:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765193.351798:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765193/real 1775765193] req@ffff90c2034bed80 x1862023741836160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765193.351828:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bed80 x1862023741836160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765193.351842:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765193.351849:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765193.351855:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765193.351942:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765193.351977:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765193.351990:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765193/real 1775765193] req@ffff90c2034bc700 x1862023741836288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765193.352003:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023741836288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765193.352011:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765193.352015:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765193.352028:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765193.352033:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836416:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765193.352049:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765193.352057:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765193/real 1775765193] req@ffff90c2034bf100 x1862023741836416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765193.352068:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023741836416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765193.352079:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765193.352082:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765193.352084:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836416:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765194.375153:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765195.399127:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765196.423120:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765197.447167:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765198.471025:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765198.471033:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765198.471037:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765198.471045:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1086 00000100:00080000:3.0:1775765198.471055:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765198.471061:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765198.471097:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741836544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765198.471120:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765198.471124:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765198.471125:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765198.471129:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1086 00000100:00080000:3.0:1775765198.471132:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765198.471135:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765198.471156:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00100000:3.0:1775765198.471156:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765198.471168:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741836672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765198.471176:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765198.471180:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775765198.471181:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741836544:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765198.471181:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765198.471186:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1086 00000100:00080000:3.0:1775765198.471190:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765198.471194:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765198.471197:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765198.471206:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741836800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765198.471215:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765198.471218:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765198.471221:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765198.471225:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1086 00000100:00080000:3.0:1775765198.471230:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765198.471234:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765198.471240:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:1.0:1775765198.471250:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765198.471255:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741836928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765198.471287:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765198/real 1775765198] req@ffff90c218da0380 x1862023741836544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765198.471302:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741836544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765198.471309:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765198.471326:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765198.471330:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741836544:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765198.471361:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765198.471377:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765198.471386:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765198/real 1775765198] req@ffff90c218da3480 x1862023741836672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765198.471395:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741836672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765198.471400:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765198.471402:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765198.471405:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765198.471408:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765198.471418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765198.471423:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765198/real 1775765198] req@ffff90c218da0700 x1862023741836800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765198.471431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741836800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765198.471436:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765198.471437:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765198.471439:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765198.471443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836928:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765198.471452:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765198.471457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765198/real 1775765198] req@ffff90c218da3100 x1862023741836928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765198.471463:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741836928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765198.471470:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765198.471472:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765198.471474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741836928:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775765199.495047:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765200.519163:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765201.542990:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765202.567011:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765203.590961:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765203.590969:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765203.590974:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765203.590982:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1092 00000100:00080000:3.0:1775765203.590992:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765203.590998:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765203.591033:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741837056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765203.591056:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765203.591059:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765203.591061:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765203.591064:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1092 00000100:00080000:3.0:1775765203.591068:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765203.591070:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765203.591078:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765203.591095:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741837184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765203.591097:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741837056:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765203.591116:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765203.591118:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765203.591120:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765203.591125:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1092 00000100:00080000:3.0:1775765203.591128:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775765203.591132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765203.591133:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765203.591138:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765203.591147:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741837312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765203.591154:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765203.591157:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1775765203.591161:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765203.591161:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765203.591169:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1092 00000100:00080000:3.0:1775765203.591174:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765203.591179:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765203.591182:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765203.591191:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741837440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765203.591206:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765203/real 1775765203] req@ffff90c218da0a80 x1862023741837056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765203.591221:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741837056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765203.591229:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765203.591232:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765203.591235:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741837056:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765203.591252:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837184:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765203.591284:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765203.591305:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765203/real 1775765203] req@ffff90c218da0000 x1862023741837184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765203.591316:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741837184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765203.591321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765203.591324:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765203.591326:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765203.591329:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765203.591342:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765203.591349:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765203/real 1775765203] req@ffff90c218da1f80 x1862023741837312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765203.591356:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741837312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765203.591361:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765203.591363:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765203.591364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765203.591375:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837440:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765203.591384:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765203.591389:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765203/real 1775765203] req@ffff90c218da3100 x1862023741837440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765203.591397:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741837440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765203.591403:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765203.591405:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765203.591407:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837440:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765204.615026:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765205.639019:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765206.663053:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765207.687140:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765208.710970:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765208.710978:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765208.710982:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765208.710990:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1097 00000100:00080000:0.0:1775765208.710999:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765208.711005:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:2.0:1775765208.711005:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765208.711053:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741837568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765208.711074:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765208.711077:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765208.711078:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765208.711081:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1097 00000100:00080000:0.0:1775765208.711083:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765208.711085:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765208.711093:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765208.711113:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741837696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765208.711122:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765208.711126:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765208.711128:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765208.711128:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741837568:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765208.711133:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1097 00000100:00080000:0.0:1775765208.711137:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765208.711141:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765208.711143:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765208.711151:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741837824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765208.711158:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765208.711160:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765208.711161:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765208.711163:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1097 00000100:00080000:0.0:1775765208.711166:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765208.711168:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765208.711171:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765208.711178:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741837952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765208.711203:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765208.711258:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765208/real 1775765208] req@ffff90c203639500 x1862023741837568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765208.711298:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741837568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765208.711309:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765208.711314:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765208.711318:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741837568:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765208.711341:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765208.711368:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765208.711382:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765208/real 1775765208] req@ffff90c203638000 x1862023741837696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765208.711396:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741837696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765208.711405:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765208.711408:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765208.711412:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765208.711418:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765208.711439:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765208.711451:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765208/real 1775765208] req@ffff90c203638700 x1862023741837824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765208.711464:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741837824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765208.711478:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765208.711482:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765208.711486:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765208.711492:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837952:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765208.711513:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765208.711524:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765208/real 1775765208] req@ffff90c203639180 x1862023741837952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765233 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765208.711537:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741837952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765233 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765208.711548:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765208.711552:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765208.711555:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741837952:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765209.735147:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765210.759003:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765211.782979:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765212.807056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765213.831024:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765213.831037:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765213.831043:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765213.831057:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1102 00000100:00080000:3.0:1775765213.831084:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775765213.831093:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765213.831095:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765213.831144:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741838080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765213.831202:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765213.831210:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765213.831214:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765213.831222:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1102 00000100:00080000:3.0:1775765213.831229:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765213.831238:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765213.831255:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765213.831270:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741838080:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765213.831287:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741838208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765213.831312:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765213.831319:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765213.831323:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765213.831328:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1102 00000100:00080000:3.0:1775765213.831347:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765213.831354:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765213.831362:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765213.831369:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765213.831394:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741838336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765213.831408:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765213.831413:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765213.831419:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765213.831421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765213/real 1775765213] req@ffff90c218da0700 x1862023741838080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765213.831429:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1102 00000100:00080000:3.0:1775765213.831440:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765213.831452:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765213.831460:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765213.831470:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741838080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765213.831480:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741838464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765213.831508:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765213.831520:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765213.831546:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741838080:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765213.831602:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838208:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765213.831631:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765213.831647:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765213/real 1775765213] req@ffff90c218da3480 x1862023741838208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765213.831662:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741838208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765213.831672:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765213.831676:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765213.831680:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765213.831691:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838336:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765213.831739:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765213.831760:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765213/real 1775765213] req@ffff90c218da0380 x1862023741838336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765213.831785:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741838336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765213.831801:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765213.831809:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765213.831815:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765213.831843:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838464:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765213.831879:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765213.831903:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765213/real 1775765213] req@ffff90c218da3800 x1862023741838464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765213.831946:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741838464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765238 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765213.831959:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765213.831963:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765213.831966:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838464:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775765214.855139:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765215.879055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765216.903100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765217.926953:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765218.950976:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765218.950984:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765218.950988:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765218.950996:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1107 00000100:00080000:0.0:1775765218.951006:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765218.951012:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765218.951045:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741838592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765218.951051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765218.951069:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765218.951072:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765218.951073:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765218.951076:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1107 00000100:00080000:0.0:1775765218.951078:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765218.951081:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765218.951089:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765218.951097:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741838720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765218.951103:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741838592:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765218.951108:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765218.951111:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765218.951112:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765218.951115:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1107 00000100:00080000:0.0:1775765218.951116:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765218.951118:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765218.951139:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765218.951151:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741838848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765218.951158:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765218.951161:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765218.951162:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765218.951164:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1107 00000100:00080000:0.0:1775765218.951168:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765218.951170:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765218.951174:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765218.951177:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765218.951184:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741838976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765218.951209:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765218/real 1775765218] req@ffff90c20363b100 x1862023741838592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765218.951222:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741838592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765218.951230:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765218.951233:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765218.951236:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741838592:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765218.951252:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765218.951265:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765218.951272:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765218/real 1775765218] req@ffff90c20363bb80 x1862023741838720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765218.951285:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741838720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765218.951289:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765218.951291:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765218.951293:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765218.951297:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765218.951306:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765218.951312:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765218/real 1775765218] req@ffff90c20363a680 x1862023741838848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765218.951319:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741838848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765218.951323:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765218.951325:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765218.951327:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765218.951335:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838976:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765218.951344:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765218.951349:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765218/real 1775765218] req@ffff90c20363aa00 x1862023741838976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765218.951355:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741838976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765243 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765218.951365:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765218.951367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765218.951369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741838976:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765219.975039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765220.999012:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765222.023019:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765223.047130:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765224.071114:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:2.0:1775765224.071114:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765224.071127:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765224.071134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765224.071148:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1112 00000100:00080000:1.0:1775765224.071161:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765224.071169:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765224.071211:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741839104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765224.071250:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765224.071256:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765224.071259:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765224.071266:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1112 00000100:00080000:1.0:1775765224.071272:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765224.071278:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765224.071287:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741839104:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775765224.071292:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765224.071317:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741839232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765224.071334:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765224.071338:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765224.071340:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765224.071345:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1112 00000100:00080000:1.0:1775765224.071350:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765224.071354:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765224.071361:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775765224.071363:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765224.071380:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741839360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765224.071396:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765224.071400:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765224.071402:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765224.071407:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765224/real 1775765224] req@ffff90c203495880 x1862023741839104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765224.071409:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1112 00000100:00080000:1.0:1775765224.071414:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765224.071422:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765224.071429:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765224.071453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741839104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765224.071458:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741839488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765224.071506:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765224.071512:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765224.071520:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741839104:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765224.071545:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765224.071570:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765224.071583:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765224/real 1775765224] req@ffff90c203497b80 x1862023741839232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765224.071596:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741839232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765224.071603:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765224.071607:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765224.071610:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765224.071616:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765224.071630:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765224.071638:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765224/real 1775765224] req@ffff90c203496300 x1862023741839360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765224.071648:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741839360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765224.071655:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765224.071658:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765224.071660:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765224.071666:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839488:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765224.071679:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765224.071686:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765224/real 1775765224] req@ffff90c203496a00 x1862023741839488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765224.071696:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741839488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765224.071705:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765224.071707:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765224.071710:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839488:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765225.095106:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765226.119106:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765227.143016:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765228.167059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765229.191071:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765229.191083:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765229.191089:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765229.191102:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1117 00000100:00080000:3.0:1775765229.191137:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765229.191146:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765229.191168:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765229.191194:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741839616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765229.191227:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765229.191232:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765229.191235:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765229.191240:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1117 00000100:00080000:3.0:1775765229.191246:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765229.191251:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765229.191263:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765229.191271:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741839616:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765229.191283:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741839744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765229.191299:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765229.191304:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765229.191307:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765229.191313:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1117 00000100:00080000:3.0:1775765229.191344:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765229.191350:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765229.191356:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765229.191371:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741839872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765229.191385:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765229.191390:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765229.191393:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765229.191400:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1117 00000100:00080000:3.0:1775765229.191406:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765229.191411:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765229.191418:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839744:192.168.202.155@tcp:38:kworker.0 00000008:00100000:3.0:1775765229.191421:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765229.191446:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741840000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765229.191506:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765229.191559:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840000:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775765230.215097:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765231.239209:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765232.263188:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765232.263219:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775765232.263265:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765232.263284:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765232.263331:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765232.263342:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765232.263354:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775765232.263381:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765229/real 1775765232] req@ffff90c218da3800 x1862023741840000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765232.263429:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741840000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765232.263446:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765232.263458:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765232.263464:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840000:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775765232.263475:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765229/real 1775765232] req@ffff90c218da1880 x1862023741839744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765232.263489:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741839744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765232.263497:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765232.263501:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765232.263505:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765232.263513:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765229/real 1775765232] req@ffff90c218da0e00 x1862023741839872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765232.263537:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741839872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765232.263548:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765232.263551:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765232.263555:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741839872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765232.263562:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765229/real 1775765232] req@ffff90c218da2d80 x1862023741839616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765232.263574:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741839616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765232.263582:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765232.263586:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765232.263590:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741839616:192.168.202.155@tcp:250:kworker.0 00010000:00010000:2.0:1775765233.287032:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765234.310958:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765234.310969:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765234.310974:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765234.311012:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1122 00000100:00080000:3.0:1775765234.311026:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765234.311035:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765234.311061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765234.311078:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741840128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765234.311106:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765234.311112:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765234.311116:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765234.311123:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1122 00000100:00080000:3.0:1775765234.311130:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765234.311136:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765234.311147:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765234.311157:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741840128:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765234.311162:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741840256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765234.311175:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765234.311179:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765234.311181:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765234.311186:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1122 00000100:00080000:3.0:1775765234.311190:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765234.311194:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765234.311200:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765234.311219:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741840384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765234.311223:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765234.311231:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765234.311235:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765234.311237:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765234.311242:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1122 00000100:00080000:3.0:1775765234.311246:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765234.311250:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765234.311256:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765234.311263:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765234/real 1775765234] req@ffff90c218da0380 x1862023741840128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765234.311269:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741840512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765234.311286:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741840128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765234.311299:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765234.311305:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765234.311311:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741840128:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765234.311339:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765234.311370:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765234.311387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765234/real 1775765234] req@ffff90c218da3480 x1862023741840256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765234.311404:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741840256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765234.311414:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765234.311419:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765234.311423:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765234.311432:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765234.311458:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765234.311482:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765234/real 1775765234] req@ffff90c218da0700 x1862023741840384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765234.311497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741840384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765234.311506:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765234.311509:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765234.311514:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765234.311522:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840512:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765234.311547:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765234.311560:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765234/real 1775765234] req@ffff90c218da3100 x1862023741840512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765234.311575:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741840512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765234.311589:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765234.311592:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765234.311597:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840512:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775765235.335106:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765236.359330:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765237.383083:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765238.406971:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765239.431070:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:2.0:1775765239.431073:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765239.431083:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765239.431091:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765239.431104:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1127 00000100:00080000:0.0:1775765239.431119:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765239.431128:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765239.431181:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741840640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765239.431224:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765239.431228:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765239.431230:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765239.431234:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1127 00000100:00080000:0.0:1775765239.431239:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765239.431243:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765239.431267:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765239.431272:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741840640:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765239.431287:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741840768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765239.431300:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765239.431305:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765239.431307:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765239.431311:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1127 00000100:00080000:0.0:1775765239.431315:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765239.431319:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765239.431324:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765239.431336:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741840896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765239.431347:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765239.431351:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765239.431352:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765239.431356:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1127 00000100:00080000:0.0:1775765239.431361:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765239.431368:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765239.431371:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765239.431378:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765239.431392:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741841024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765239.431425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765239/real 1775765239] req@ffff90c203638380 x1862023741840640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765239.431450:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741840640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765239.431462:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765239.431467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765239.431472:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741840640:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765239.431498:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765239.431520:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765239.431531:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765239/real 1775765239] req@ffff90c203638e00 x1862023741840768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765239.431545:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741840768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765239.431553:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765239.431556:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765239.431560:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765239.431565:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840896:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765239.431580:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765239.431588:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765239/real 1775765239] req@ffff90c20363b480 x1862023741840896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765239.431599:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741840896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765239.431606:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765239.431609:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765239.431612:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741840896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765239.431617:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841024:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765239.431631:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765239.431639:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765239/real 1775765239] req@ffff90c20363b800 x1862023741841024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765239.431649:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741841024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765239.431666:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765239.431669:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765239.431672:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841024:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765240.455047:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765241.478972:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765242.503022:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765243.527006:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765244.550968:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765244.550975:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765244.550980:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765244.550988:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1133 00010000:00010000:2.0:1775765244.550992:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765244.550997:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765244.551003:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765244.551037:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741841152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765244.551059:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765244.551062:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765244.551063:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765244.551066:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1133 00000100:00080000:0.0:1775765244.551068:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765244.551071:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765244.551078:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765244.551089:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741841280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765244.551091:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741841152:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765244.551110:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765244.551112:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765244.551113:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765244.551116:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1133 00000100:00080000:0.0:1775765244.551119:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765244.551121:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765244.551123:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765244.551132:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741841408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765244.551138:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765244.551140:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765244.551141:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765244.551143:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1133 00000100:00080000:0.0:1775765244.551146:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765244.551149:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765244.551151:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765244.551158:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741841536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765244.551165:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765244.551200:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765244/real 1775765244] req@ffff90c203639f80 x1862023741841152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765244.551216:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741841152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765244.551224:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765244.551228:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765244.551232:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741841152:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765244.551250:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765244.551267:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765244.551275:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765244/real 1775765244] req@ffff90c20363b800 x1862023741841280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765244.551293:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741841280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765244.551303:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765244.551305:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765244.551308:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765244.551311:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765244.551324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765244.551333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765244/real 1775765244] req@ffff90c20363b480 x1862023741841408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765244.551342:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741841408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765244.551347:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765244.551349:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765244.551351:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765244.551354:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841536:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765244.551365:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765244.551373:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765244/real 1775765244] req@ffff90c203638e00 x1862023741841536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765244.551380:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741841536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765244.551387:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765244.551388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765244.551391:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841536:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775765245.575010:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765246.599037:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765247.622971:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765248.647038:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765249.670995:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765249.671007:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765249.671014:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:0.0:1775765249.671022:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765249.671026:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1138 00000100:00080000:3.0:1775765249.671041:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765249.671048:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765249.671111:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741841664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765249.671139:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765249.671159:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765249.671162:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765249.671167:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1138 00000100:00080000:3.0:1775765249.671171:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765249.671175:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765249.671189:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765249.671204:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741841792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765249.671230:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741841664:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765249.671232:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765249.671239:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765249.671241:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765249.671245:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1138 00000100:00080000:3.0:1775765249.671247:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765249.671250:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765249.671254:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765249.671265:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741841920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765249.671272:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765249.671274:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765249.671276:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765249.671280:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1138 00000100:00080000:3.0:1775765249.671283:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765249.671285:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765249.671296:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765249.671307:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741842048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765249.671321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765249.671368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765249/real 1775765249] req@ffff90c218da1f80 x1862023741841664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765249.671385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741841664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765249.671395:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765249.671400:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765249.671403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741841664:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765249.671427:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765249.671443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765249.671451:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765249/real 1775765249] req@ffff90c218da0000 x1862023741841792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765249.671460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741841792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765249.671466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765249.671468:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765249.671470:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765249.671477:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765249.671488:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765249.671494:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765249/real 1775765249] req@ffff90c218da0a80 x1862023741841920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765249.671502:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741841920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765249.671508:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765249.671510:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765249.671512:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741841920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765249.671516:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842048:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765249.671539:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765249.671551:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765249/real 1775765249] req@ffff90c218da3100 x1862023741842048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765249.671562:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741842048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765249.671571:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765249.671573:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765249.671575:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842048:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775765250.695006:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765251.718949:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765252.743021:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765253.767025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765254.790973:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765254.790983:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765254.790987:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765254.790998:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1143 00010000:00010000:2.0:1775765254.791001:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765254.791008:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765254.791015:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765254.791054:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741842176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765254.791077:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765254.791081:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765254.791083:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765254.791087:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1143 00000100:00080000:3.0:1775765254.791091:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765254.791096:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765254.791105:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765254.791110:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741842176:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765254.791117:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741842304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765254.791126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765254.791129:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765254.791131:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765254.791134:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1143 00000100:00080000:3.0:1775765254.791137:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765254.791141:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765254.791145:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765254.791153:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741842432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765254.791161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765254.791163:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765254.791165:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765254.791167:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765254.791169:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1143 00000100:00080000:3.0:1775765254.791172:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765254.791175:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765254.791179:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765254.791188:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741842560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765254.791204:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765254/real 1775765254] req@ffff90c218da0700 x1862023741842176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765254.791222:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741842176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765254.791232:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765254.791236:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765254.791241:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741842176:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765254.791281:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842304:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765254.791306:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765254.791319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765254/real 1775765254] req@ffff90c218da3480 x1862023741842304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765254.791331:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741842304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765254.791338:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765254.791341:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765254.791344:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765254.791350:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842432:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765254.791370:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765254.791380:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765254/real 1775765254] req@ffff90c218da0380 x1862023741842432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765254.791398:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741842432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765254.791406:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765254.791408:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765254.791411:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765254.791417:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842560:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765254.791438:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765254.791448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765254/real 1775765254] req@ffff90c218da2d80 x1862023741842560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765279 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765254.791456:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741842560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765279 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765254.791464:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765254.791466:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765254.791468:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842560:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765255.815065:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765256.839039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765257.862977:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765258.887076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765259.911024:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765259.911039:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765259.911046:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765259.911061:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1148 00000100:00080000:3.0:1775765259.911078:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:2.0:1775765259.911081:0:18307:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 4/4 locks to regular blp list, 1 blwis in pool 00000100:00080000:3.0:1775765259.911088:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:2.0:1775765259.911121:0:10909:0:(ldlm_lockd.c:2794:ldlm_bl_get_work()) Got 4 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:2.0:1775765259.911132:0:10909:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c209553200/0xfb95bb0df4166dbb lrc: 2/0,0 mode: PR/PR res: [0x200000403:0x10:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x721ffa0b6629a7be expref: -99 pid: 18302 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775765259.911142:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741842688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765259.911178:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765259.911181:0:10909:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c209552800/0xfb95bb0df4166dad lrc: 2/0,0 mode: CR/CR res: [0x200000403:0x10:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x721ffa0b6629a7a9 expref: -99 pid: 18302 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000100:00080000:3.0:1775765259.911200:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765259.911205:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765259.911209:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765259.911214:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1148 00000100:00080000:3.0:1775765259.911219:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765259.911223:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765259.911236:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000080:00200000:2.0:1775765259.911248:0:10909:0:(vvp_object.c:117:vvp_conf_set()) [0x200000403:0x10:0x0]: losing layout lock 00000100:00100000:3.0:1775765259.911252:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741842816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765259.911268:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765259.911272:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765259.911274:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765259.911280:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1148 00000100:00080000:3.0:1775765259.911284:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765259.911290:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765259.911299:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00010000:00010000:2.0:1775765259.911302:0:10909:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c218fec600/0xfb95bb0df4166da6 lrc: 2/0,0 mode: CW/CW res: [0x200000403:0x10:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x721ffa0b6629a794 expref: -99 pid: 18302 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775765259.911313:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741842944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765259.911320:0:10909:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c2027d0a00/0xfb95bb0df4166bd1 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x1:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x721ffa0b66299e1e expref: -99 pid: 12293 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00000100:00080000:3.0:1775765259.911328:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765259.911333:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765259.911335:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765259.911341:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1148 00000100:00080000:3.0:1775765259.911347:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765259.911359:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:2.0:1775765259.911361:0:10909:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c209553200/0xfb95bb0df4166dbb lrc: 1/0,0 mode: --/PR res: [0x200000403:0x10:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x721ffa0b6629a7be expref: -99 pid: 18302 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000100:00100000:0.0:1775765259.911362:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741842688:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775765259.911367:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00010000:00010000:2.0:1775765259.911394:0:10909:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c209552800/0xfb95bb0df4166dad lrc: 1/0,0 mode: --/CR res: [0x200000403:0x10:0x0].0x0 bits 0x8/0x0 rrc: 4 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x721ffa0b6629a7a9 expref: -99 pid: 18302 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775765259.911401:0:10909:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c218fec600/0xfb95bb0df4166da6 lrc: 1/0,0 mode: --/CW res: [0x200000403:0x10:0x0].0x0 bits 0x5/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x721ffa0b6629a794 expref: -99 pid: 18302 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000100:00100000:3.0:1775765259.911403:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741843200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1775765259.911412:0:10909:0:(ldlm_request.c:1438:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c2027d0a00/0xfb95bb0df4166bd1 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x1:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x721ffa0b66299e1e expref: -99 pid: 12293 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775765259.911433:0:10909:0:(ldlm_request.c:1490:ldlm_cancel_pack()) 4 locks packed 00010000:00010000:2.0:1775765259.911461:0:10909:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c209553200/0xfb95bb0df4166dbb lrc: 0/0,0 mode: --/PR res: [0x200000403:0x10:0x0].0x0 bits 0x20/0x0 rrc: 4 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x721ffa0b6629a7be expref: -99 pid: 18302 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1775765259.911479:0:10909:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c209552800/0xfb95bb0df4166dad lrc: 0/0,0 mode: --/CR res: [0x200000403:0x10:0x0].0x0 bits 0x8/0x0 rrc: 3 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x721ffa0b6629a7a9 expref: -99 pid: 18302 timeout: 0 lvb_type: 3 lru_score: 2 lru_type: 0 00000800:00000100:0.0:1775765259.911496:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00010000:00010000:2.0:1775765259.911496:0:10909:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c218fec600/0xfb95bb0df4166da6 lrc: 0/0,0 mode: --/CW res: [0x200000403:0x10:0x0].0x0 bits 0x5/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x721ffa0b6629a794 expref: -99 pid: 18302 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00000100:00080000:3.0:1775765259.911504:0:2428:0:(client.c:1830:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != CONNECTING) req@ffff90c2034bdc00 x1862023741843072/t0(0) o103->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:17/18 lens 344/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:2.0:1775765259.911547:0:10909:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff90c203172800 lock: ffff90c2027d0a00/0xfb95bb0df4166bd1 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x1:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x721ffa0b66299e1e expref: -99 pid: 12293 timeout: 0 lvb_type: 0 lru_score: 2 lru_type: 0 00010000:00010000:2.0:1775765259.911579:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765259.911582:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765259.911588:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765259.911590:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765259/real 1775765259] req@ffff90c218da0e00 x1862023741842688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765259.911626:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741842688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765259.911646:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765259.911655:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765259.911662:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741842688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765259.911678:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765259.911727:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765259.911778:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765259/real 1775765259] req@ffff90c218da1880 x1862023741842816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765259.911804:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741842816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765259.911821:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765259.911826:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765259.911831:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765259.911840:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765259.911926:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765259.911946:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765259/real 1775765259] req@ffff90c218da3800 x1862023741842944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765259.911971:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741842944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765259.911981:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765259.911985:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765259.911989:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741842944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765259.912024:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843200:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765259.912049:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765259.912060:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765259/real 1775765259] req@ffff90c218da2d80 x1862023741843200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765259.912075:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741843200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765284 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765259.912086:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765259.912090:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765259.912093:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843200:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765260.935082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765261.959067:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765262.982952:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765264.007141:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765265.031035:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765265.031063:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765265.031069:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765265.031081:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1153 00000100:00080000:0.0:1775765265.031096:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765265.031106:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765265.031115:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765265.031160:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741843328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765265.031211:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765265.031216:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765265.031219:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765265.031223:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1153 00000100:00080000:0.0:1775765265.031228:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765265.031232:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765265.031244:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765265.031260:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741843456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765265.031275:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765265.031280:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765265.031284:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765265.031288:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741843328:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765265.031294:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1153 00000100:00080000:0.0:1775765265.031306:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765265.031310:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765265.031316:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765265.031334:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741843584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765265.031347:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765265.031351:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765265.031354:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765265.031359:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1153 00000100:00080000:0.0:1775765265.031364:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765265.031370:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765265.031376:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765265.031386:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765265.031393:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741843712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765265.031455:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765265/real 1775765265] req@ffff90c203638380 x1862023741843328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765265.031474:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741843328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765265.031484:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765265.031489:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765265.031493:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741843328:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765265.031515:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765265.031535:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765265.031546:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765265/real 1775765265] req@ffff90c20363aa00 x1862023741843456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765265.031557:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741843456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765265.031570:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765265.031573:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765265.031576:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765265.031581:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843584:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765265.031594:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765265.031601:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765265/real 1775765265] req@ffff90c20363a680 x1862023741843584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765265.031610:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741843584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765265.031638:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765265.031641:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765265.031644:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765265.031650:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843712:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765265.031665:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765265.031673:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765265/real 1775765265] req@ffff90c20363bb80 x1862023741843712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765265.031684:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741843712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765265.031693:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765265.031695:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765265.031717:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843712:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775765266.055024:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765267.078992:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765268.103077:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765269.126946:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765270.150990:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765270.150999:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765270.151003:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765270.151012:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1158 00000100:00080000:0.0:1775765270.151023:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765270.151030:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765270.151057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765270.151068:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741843840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765270.151092:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765270.151095:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765270.151097:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765270.151100:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1158 00000100:00080000:0.0:1775765270.151103:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765270.151107:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765270.151115:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765270.151127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741843968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765270.151132:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741843840:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765270.151150:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765270.151153:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765270.151154:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765270.151159:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1158 00000100:00080000:0.0:1775765270.151181:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765270.151185:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765270.151191:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765270.151205:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741844096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765270.151215:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765270.151219:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765270.151220:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765270.151224:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1158 00000800:00000100:2.0:1775765270.151232:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765270.151234:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765270.151241:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765270.151248:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765270.151259:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741844224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765270.151283:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765270/real 1775765270] req@ffff90c20363b100 x1862023741843840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765270.151306:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741843840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765270.151334:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765270.151338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765270.151341:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741843840:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765270.151361:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765270.151378:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765270.151387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765270/real 1775765270] req@ffff90c203639180 x1862023741843968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765270.151396:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741843968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765270.151403:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765270.151405:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765270.151408:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741843968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765270.151412:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765270.151424:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765270.151430:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765270/real 1775765270] req@ffff90c203638700 x1862023741844096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765270.151439:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741844096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765270.151445:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765270.151447:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765270.151449:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765270.151453:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844224:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765270.151463:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765270.151469:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765270/real 1775765270] req@ffff90c203638000 x1862023741844224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765270.151477:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741844224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765270.151485:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765270.151487:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765270.151489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844224:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765271.175042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775765272.199034:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765273.223139:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765274.247127:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765275.271082:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765275.271092:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765275.271100:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765275.271105:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765275.271114:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1163 00000100:00080000:3.0:1775765275.271124:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765275.271130:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765275.271166:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741844352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765275.271192:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765275.271195:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765275.271197:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765275.271200:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1163 00000100:00080000:3.0:1775765275.271203:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765275.271205:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765275.271213:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765275.271224:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741844480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765275.271233:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765275.271239:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741844352:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765275.271239:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765275.271242:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765275.271250:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1163 00000100:00080000:3.0:1775765275.271253:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765275.271256:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765275.271259:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765275.271269:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741844608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765275.271276:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765275.271279:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765275.271280:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765275.271283:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1163 00000100:00080000:3.0:1775765275.271285:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765275.271288:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765275.271291:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765275.271300:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741844736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765275.271324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765275.271364:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765275/real 1775765275] req@ffff90c218da0380 x1862023741844352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765275.271394:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741844352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765275.271406:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765275.271410:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765275.271415:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741844352:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765275.271440:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765275.271466:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765275.271500:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765275/real 1775765275] req@ffff90c218da3480 x1862023741844480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765275.271521:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741844480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765275.271561:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765275.271566:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765275.271571:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765275.271579:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765275.271662:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765275.271675:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765275/real 1775765275] req@ffff90c218da0700 x1862023741844608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765275.271688:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741844608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765275.271704:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765275.271725:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765275.271730:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765275.271740:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844736:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765275.271769:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765275.271784:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765275/real 1775765275] req@ffff90c218da3100 x1862023741844736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765275.271802:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741844736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765275.271818:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765275.271823:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765275.271829:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844736:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765276.294981:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765277.319072:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765278.343049:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765279.367127:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765280.390970:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765280.390983:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765280.390991:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765280.391004:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1168 00000100:00080000:3.0:1775765280.391039:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765280.391050:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765280.391129:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765280.391132:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741844864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765280.391178:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765280.391182:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765280.391184:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765280.391188:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1168 00000100:00080000:3.0:1775765280.391192:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765280.391197:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765280.391209:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765280.391235:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741844992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765280.391238:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741844864:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765280.391257:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765280.391261:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765280.391262:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765280.391266:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1168 00000100:00080000:3.0:1775765280.391268:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765280.391274:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765280.391280:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765280.391297:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741845120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765280.391310:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765280.391314:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765280.391316:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765280.391325:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1168 00000100:00080000:3.0:1775765280.391331:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765280.391337:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765280.391342:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765280.391348:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765280.391402:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741845248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765280.391448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765280/real 1775765280] req@ffff90c218da0a80 x1862023741844864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765280.391475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741844864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765280.391489:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765280.391495:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765280.391501:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741844864:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765280.391535:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844992:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765280.391606:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765280.391619:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765280/real 1775765280] req@ffff90c218da0000 x1862023741844992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765280.391631:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741844992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765280.391639:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765280.391643:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765280.391646:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741844992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765280.391661:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765280.391677:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765280.391685:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765280/real 1775765280] req@ffff90c218da1f80 x1862023741845120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765280.391695:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741845120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765280.391702:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765280.391705:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765280.391707:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765280.391712:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845248:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765280.391726:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765280.391733:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765280/real 1775765280] req@ffff90c218da3100 x1862023741845248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765280.391744:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741845248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765280.391764:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765280.391766:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765280.391769:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845248:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765281.415169:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765282.439029:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765283.463021:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765284.487049:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765285.510969:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765285.510978:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765285.510981:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765285.510990:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1173 00000100:00080000:3.0:1775765285.511000:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765285.511007:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765285.511034:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765285.511050:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741845376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765285.511068:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765285.511071:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765285.511072:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765285.511074:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1173 00000100:00080000:3.0:1775765285.511077:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765285.511079:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765285.511088:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765285.511097:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741845504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765285.511101:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741845376:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765285.511109:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765285.511112:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765285.511114:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765285.511117:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1173 00000100:00080000:3.0:1775765285.511120:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765285.511122:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765285.511126:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765285.511133:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741845632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765285.511139:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765285.511141:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765285.511142:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765285.511145:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1173 00000100:00080000:3.0:1775765285.511157:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765285.511159:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765285.511163:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765285.511164:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765285.511175:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741845760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765285.511199:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765285/real 1775765285] req@ffff90c218da0700 x1862023741845376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765285.511214:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741845376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765285.511221:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765285.511224:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765285.511227:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741845376:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765285.511244:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765285.511257:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765285.511265:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765285/real 1775765285] req@ffff90c218da3480 x1862023741845504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765285.511272:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741845504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765285.511282:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765285.511284:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765285.511286:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765285.511289:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765285.511299:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765285.511304:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765285/real 1775765285] req@ffff90c218da0380 x1862023741845632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765285.511310:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741845632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765285.511315:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765285.511317:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765285.511319:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765285.511322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845760:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765285.511331:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765285.511336:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765285/real 1775765285] req@ffff90c218da2d80 x1862023741845760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765285.511342:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741845760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765285.511349:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765285.511351:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765285.511353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741845760:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765286.535019:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765287.559050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765288.583029:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765289.607031:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765290.630972:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765290.630990:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765290.630997:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765290.631015:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1179 00000100:00080000:3.0:1775765290.631031:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765290.631041:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765290.631061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765290.631099:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741845888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765290.631132:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765290.631137:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765290.631139:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765290.631146:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1179 00000100:00080000:3.0:1775765290.631151:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765290.631156:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765290.631165:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741845888:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775765290.631169:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765290.631187:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741846016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765290.631199:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765290.631204:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765290.631206:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765290.631210:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1179 00000100:00080000:3.0:1775765290.631231:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765290.631236:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765290.631243:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765290.631243:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765290.631259:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741846144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765290.631271:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765290.631276:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765290.631278:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765290.631284:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1179 00000100:00100000:2.0:1775765290.631287:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765290/real 1775765290] req@ffff90c218da3800 x1862023741845888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765290.631291:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765290.631302:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765290.631309:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741845888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775765290.631309:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775765290.631322:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765290.631327:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:3.0:1775765290.631331:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741846272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765290.631333:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741845888:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765290.631362:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765290.631379:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765290.631387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765290/real 1775765290] req@ffff90c218da1880 x1862023741846016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765290.631395:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741846016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765290.631401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765290.631403:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765290.631405:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765290.631408:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765290.631423:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765290.631434:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765290/real 1775765290] req@ffff90c218da0e00 x1862023741846144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765290.631447:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741846144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765290.631456:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765290.631489:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765290.631493:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765290.631498:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846272:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765290.631515:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765290.631523:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765290/real 1775765290] req@ffff90c218da2d80 x1862023741846272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765290.631531:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741846272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765290.631539:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765290.631541:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765290.631544:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846272:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765291.655052:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765292.679229:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765293.703123:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765294.727165:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765295.751079:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765295.751092:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765295.751100:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:1.0:1775765295.751111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765295.751113:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1184 00000100:00080000:0.0:1775765295.751131:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765295.751142:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765295.751188:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741846400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765295.751227:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765295.751232:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765295.751234:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765295.751239:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1184 00000100:00080000:0.0:1775765295.751243:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765295.751248:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765295.751259:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765295.751277:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741846528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765295.751294:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765295.751298:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765295.751301:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765295.751309:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1184 00000100:00080000:0.0:1775765295.751315:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765295.751315:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741846400:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765295.751324:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765295.751344:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765295.751362:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741846656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765295.751381:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765295.751388:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765295.751391:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765295.751400:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1184 00000100:00080000:0.0:1775765295.751407:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765295.751440:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765295.751460:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765295.751485:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741846784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765295.751529:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765295.751574:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765295.751602:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846784:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765296.775219:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765297.799058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765298.823111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765298.823171:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775765298.823213:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765298.823229:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765298.823266:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765298.823277:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765298.823288:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775765298.823307:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765295/real 1775765298] req@ffff90c203638000 x1862023741846528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765298.823334:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741846528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765298.823346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765298.823351:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765298.823355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765298.823362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765295/real 1775765298] req@ffff90c203638700 x1862023741846656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765298.823373:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741846656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765298.823379:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765298.823382:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765298.823426:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765298.823432:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765295/real 1775765298] req@ffff90c203639180 x1862023741846784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765298.823448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741846784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765298.823458:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765298.823461:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765298.823464:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741846784:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775765298.823470:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765295/real 1775765298] req@ffff90c203639500 x1862023741846400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765320 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765298.823481:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741846400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765320 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765298.823487:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765298.823489:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765298.823492:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741846400:192.168.202.155@tcp:250:kworker.0 00010000:00010000:0.0:1775765299.847045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765300.870977:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765300.870986:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765300.870991:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765300.871001:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1189 00000100:00080000:0.0:1775765300.871012:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765300.871019:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765300.871022:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765300.871063:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741846912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765300.871087:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765300.871091:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765300.871092:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765300.871095:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1189 00000100:00080000:0.0:1775765300.871098:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765300.871101:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765300.871110:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765300.871122:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741847040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765300.871132:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765300.871153:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765300.871154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741846912:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765300.871156:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765300.871168:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1189 00000100:00080000:0.0:1775765300.871172:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765300.871175:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765300.871180:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765300.871193:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741847168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765300.871219:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765300.871223:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765300.871224:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765300.871227:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1189 00000100:00080000:0.0:1775765300.871232:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765300.871237:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765300.871244:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765300.871258:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741847296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765300.871267:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765300.871319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765300/real 1775765300] req@ffff90c20363b100 x1862023741846912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765300.871343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741846912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765300.871357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765300.871362:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765300.871385:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741846912:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765300.871416:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765300.871447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765300.871463:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765300/real 1775765300] req@ffff90c20363bb80 x1862023741847040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765300.871479:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741847040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765300.871489:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765300.871494:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765300.871498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765300.871507:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765300.871533:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765300.871548:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765300/real 1775765300] req@ffff90c20363a680 x1862023741847168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765300.871566:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741847168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765300.871575:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765300.871580:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765300.871584:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765300.871593:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847296:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765300.871620:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765300.871634:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765300/real 1775765300] req@ffff90c20363aa00 x1862023741847296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765300.871649:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741847296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765325 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765300.871662:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765300.871666:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765300.871671:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847296:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765301.895072:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765302.919033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765303.943041:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765304.967055:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765305.991043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765305.991061:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765305.991074:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765305.991082:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765305.991097:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1194 00000100:00080000:1.0:1775765305.991115:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765305.991123:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765305.991177:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741847424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765305.991220:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765305.991227:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765305.991230:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765305.991237:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1194 00000100:00080000:1.0:1775765305.991243:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765305.991250:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765305.991265:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765305.991270:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741847424:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765305.991290:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741847552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765305.991307:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765305.991313:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765305.991345:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765305.991353:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1194 00000100:00080000:1.0:1775765305.991360:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765305.991377:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765305.991386:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765305.991406:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765305.991409:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741847680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765305.991426:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765305.991430:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765305.991433:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765305.991438:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1194 00000100:00080000:1.0:1775765305.991442:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765305.991447:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765305.991478:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765305/real 1775765305] req@ffff90c203496680 x1862023741847424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765305.991479:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765305.991498:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741847808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765305.991517:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741847424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765305.991535:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765305.991543:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765305.991549:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741847424:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765305.991597:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847552:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765305.991631:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765305.991657:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765305/real 1775765305] req@ffff90c203494a80 x1862023741847552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765305.991671:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741847552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765305.991679:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765305.991683:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765305.991686:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765305.991692:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765305.991710:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765305.991720:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765305/real 1775765305] req@ffff90c203495180 x1862023741847680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765305.991730:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741847680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765305.991738:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765305.991740:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765305.991743:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765305.991748:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847808:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765305.991770:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765305.991779:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765305/real 1775765305] req@ffff90c203495500 x1862023741847808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765305.991789:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741847808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765330 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765305.991799:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765305.991801:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765305.991804:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741847808:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765307.015083:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765308.039101:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765309.063007:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765310.087099:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765311.111115:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765311.111131:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765311.111138:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765311.111155:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1199 00000100:00080000:2.0:1775765311.111173:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765311.111184:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765311.111246:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023741847936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765311.111331:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765311.111335:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765311.111336:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765311.111340:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1199 00000100:00080000:2.0:1775765311.111343:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765311.111346:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765311.111356:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765311.111367:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023741848064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765311.111375:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765311.111378:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765311.111380:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765311.111385:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1199 00000100:00100000:0.0:1775765311.111388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741847936:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775765311.111388:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765311.111392:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765311.111399:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765311.111406:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023741848192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765311.111413:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765311.111415:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765311.111416:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765311.111420:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1199 00000100:00080000:2.0:1775765311.111422:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765311.111425:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765311.111428:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765311.111436:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023741848320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765311.111479:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00010000:00010000:3.0:1775765311.111525:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765311.111532:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765311/real 1775765311] req@ffff90c2034bdf80 x1862023741847936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765311.111557:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023741847936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765311.111570:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765311.111575:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765311.111578:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741847936:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765311.111621:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765311.111653:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765311.111669:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765311/real 1775765311] req@ffff90c2034bd180 x1862023741848064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765311.111684:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023741848064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765311.111695:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765311.111699:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765311.111703:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765311.111710:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765311.111756:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765311.111767:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765311/real 1775765311] req@ffff90c2034bc380 x1862023741848192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765311.111781:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023741848192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765311.111790:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765311.111793:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765311.111797:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765311.111803:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848320:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765311.111821:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765311.111831:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765311/real 1775765311] req@ffff90c2034bce00 x1862023741848320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765311.111843:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023741848320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765311.111855:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765311.111858:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765311.111863:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848320:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765312.135121:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765313.159123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765314.183154:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765315.207038:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765316.230975:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765316.230986:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765316.230992:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765316.231001:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1204 00000100:00080000:1.0:1775765316.231010:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765316.231017:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765316.231024:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765316.231053:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741848448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765316.231075:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765316.231078:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765316.231079:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765316.231082:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1204 00000100:00080000:1.0:1775765316.231085:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765316.231088:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765316.231097:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765316.231102:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741848448:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765316.231114:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741848576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765316.231127:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765316.231131:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765316.231134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765316.231138:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1204 00000100:00080000:1.0:1775765316.231151:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765316.231154:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:0.0:1775765316.231158:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775765316.231161:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765316.231173:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741848704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765316.231181:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765316.231183:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765316.231185:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765316.231188:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1204 00000100:00080000:1.0:1775765316.231191:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765316.231193:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765316.231198:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765316.231206:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765316/real 1775765316] req@ffff90c203494700 x1862023741848448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765316.231208:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741848832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765316.231253:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741848448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765316.231261:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765316.231265:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765316.231268:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741848448:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765316.231286:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765316.231301:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765316.231309:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765316/real 1775765316] req@ffff90c203495500 x1862023741848576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765316.231317:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741848576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765316.231322:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765316.231324:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765316.231326:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765316.231330:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765316.231340:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765316.231345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765316/real 1775765316] req@ffff90c203495180 x1862023741848704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765316.231352:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741848704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765316.231357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765316.231359:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765316.231360:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765316.231364:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848832:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765316.231373:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765316.231378:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765316/real 1775765316] req@ffff90c203494a80 x1862023741848832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765316.231385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741848832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765316.231391:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765316.231393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765316.231395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741848832:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765317.255048:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765318.279160:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765319.303116:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765320.327052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765321.351021:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765321.351041:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765321.351050:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765321.351067:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1209 00000100:00080000:0.0:1775765321.351086:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765321.351098:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765321.351131:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765321.351211:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741848960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765321.351270:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765321.351278:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765321.351283:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765321.351293:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1209 00000100:00080000:0.0:1775765321.351303:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765321.351311:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765321.351327:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765321.351331:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741848960:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765321.351355:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741849088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765321.351372:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765321.351378:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765321.351381:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765321.351386:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1209 00000100:00080000:0.0:1775765321.351391:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765321.351395:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765321.351401:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765321.351416:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741849216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765321.351430:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765321.351435:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765321.351437:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765321.351442:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1209 00000100:00080000:0.0:1775765321.351446:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765321.351451:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765321.351462:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765321.351462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765321.351484:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741849344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765321.351563:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765321/real 1775765321] req@ffff90c203638380 x1862023741848960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765321.351605:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741848960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765321.351627:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765321.351635:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765321.351644:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741848960:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765321.351695:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849088:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765321.351732:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765321.351778:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765321/real 1775765321] req@ffff90c203638e00 x1862023741849088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765321.351795:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741849088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765321.351807:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765321.351812:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765321.351816:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765321.351824:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849216:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765321.351846:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765321.351858:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765321/real 1775765321] req@ffff90c20363b480 x1862023741849216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765321.351916:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741849216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765321.351927:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765321.351931:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765321.351935:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765321.351942:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849344:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765321.351966:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765321.351978:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765321/real 1775765321] req@ffff90c20363b800 x1862023741849344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765321.351992:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741849344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765321.352005:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765321.352008:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765321.352011:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849344:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765322.375047:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765323.398965:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765324.423077:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765325.447055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765326.470965:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765326.470978:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765326.470985:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765326.471006:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1214 00000100:00080000:0.0:1775765326.471024:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765326.471035:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765326.471090:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741849472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765326.471119:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765326.471123:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765326.471140:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775765326.471146:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765326.471148:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1214 00000100:00080000:0.0:1775765326.471156:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765326.471161:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765326.471176:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765326.471199:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741849600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765326.471215:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741849472:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765326.471226:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765326.471233:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765326.471236:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765326.471245:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1214 00000100:00080000:0.0:1775765326.471250:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765326.471257:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765326.471265:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765326.471283:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741849728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765326.471292:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765326.471305:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765326.471307:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765326.471311:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1214 00000100:00080000:0.0:1775765326.471315:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765326.471322:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765326.471323:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765326.471339:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765326.471359:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741849856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765326.471394:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765326/real 1775765326] req@ffff90c203639f80 x1862023741849472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765326.471419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741849472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765326.471433:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765326.471439:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765326.471444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741849472:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765326.471475:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765326.471501:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765326.471515:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765326/real 1775765326] req@ffff90c20363b800 x1862023741849600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765326.471530:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741849600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765326.471540:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765326.471544:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765326.471548:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765326.471555:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765326.471573:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765326.471583:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765326/real 1775765326] req@ffff90c20363b480 x1862023741849728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765326.471606:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741849728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765326.471615:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765326.471619:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765326.471622:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765326.471629:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849856:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765326.471647:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765326.471665:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765326/real 1775765326] req@ffff90c203638e00 x1862023741849856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765326.471678:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741849856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765326.471690:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765326.471694:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765326.471698:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741849856:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765327.495015:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765328.519040:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765329.543025:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765330.567019:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765331.591042:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765331.591055:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765331.591062:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765331.591076:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1220 00000100:00080000:1.0:1775765331.591111:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765331.591122:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765331.591126:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765331.591174:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741849984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765331.591212:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765331.591218:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765331.591220:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765331.591225:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1220 00000100:00080000:1.0:1775765331.591230:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765331.591235:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765331.591247:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765331.591263:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741850112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765331.591279:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741849984:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765331.591282:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765331.591296:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765331.591298:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765331.591304:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1220 00000100:00080000:1.0:1775765331.591308:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765331.591312:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765331.591319:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765331.591334:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741850240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765331.591346:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765331.591349:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765331.591352:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765331.591356:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1220 00000100:00080000:1.0:1775765331.591362:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765331.591366:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765331.591372:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765331.591381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765331.591395:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741850368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765331.591439:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765331/real 1775765331] req@ffff90c203496680 x1862023741849984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765331.591465:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741849984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765331.591486:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765331.591492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765331.591498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741849984:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765331.591525:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765331.591549:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765331.591563:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765331/real 1775765331] req@ffff90c203496a00 x1862023741850112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765331.591578:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741850112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765331.591588:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765331.591592:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765331.591596:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765331.591603:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850240:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765331.591621:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765331.591631:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765331/real 1775765331] req@ffff90c203496300 x1862023741850240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765331.591644:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741850240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765331.591653:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765331.591657:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765331.591660:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765331.591668:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850368:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765331.591685:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765331.591695:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765331/real 1775765331] req@ffff90c203497b80 x1862023741850368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765331.591708:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741850368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765331.591720:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765331.591723:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765331.591726:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850368:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765332.615190:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765333.639115:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765334.663043:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765335.687172:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765336.711011:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765336.711026:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765336.711053:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765336.711072:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1225 00000100:00080000:1.0:1775765336.711092:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775765336.711095:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765336.711105:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765336.711175:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741850496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765336.711214:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765336.711219:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765336.711221:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765336.711226:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1225 00000100:00080000:1.0:1775765336.711230:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765336.711236:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765336.711250:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765336.711292:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741850624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765336.711292:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741850496:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765336.711316:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765336.711321:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765336.711323:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765336.711328:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1225 00000100:00080000:1.0:1775765336.711333:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765336.711338:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765336.711344:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765336.711361:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741850752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765336.711372:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765336.711376:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765336.711378:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765336.711383:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1225 00000100:00080000:1.0:1775765336.711389:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765336.711393:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765336.711400:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765336.711406:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765336.711420:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741850880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765336.711469:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765336/real 1775765336] req@ffff90c203495880 x1862023741850496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765336.711495:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741850496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765336.711508:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765336.711514:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765336.711519:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741850496:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765336.711551:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765336.711575:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765336.711588:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765336/real 1775765336] req@ffff90c203495c00 x1862023741850624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765336.711604:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741850624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765336.711613:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765336.711616:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765336.711620:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765336.711627:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765336.711644:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765336.711670:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765336/real 1775765336] req@ffff90c203494380 x1862023741850752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765336.711685:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741850752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765336.711695:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765336.711698:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765336.711701:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765336.711708:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850880:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765336.711758:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765336.711778:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765336/real 1775765336] req@ffff90c203496d80 x1862023741850880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765361 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765336.711812:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741850880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765361 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765336.711851:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765336.711857:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765336.711862:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741850880:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765337.735211:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765338.759170:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765339.783166:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765340.807081:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765341.831143:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765341.831160:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765341.831168:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775765341.831172:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765341.831186:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1230 00000100:00080000:2.0:1775765341.831207:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765341.831223:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765341.831296:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023741851008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765341.831336:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765341.831341:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765341.831343:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765341.831348:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1230 00000100:00080000:2.0:1775765341.831352:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765341.831356:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765341.831377:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765341.831392:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023741851136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765341.831405:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765341.831409:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765341.831412:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765341.831418:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1230 00000100:00080000:2.0:1775765341.831422:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765341.831429:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765341.831436:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741851008:192.168.202.155@tcp:250:kworker.0 00000008:00100000:2.0:1775765341.831438:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765341.831454:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023741851264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765341.831465:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765341.831469:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775765341.831472:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765341.831477:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1230 00000100:00080000:2.0:1775765341.831481:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765341.831487:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765341.831492:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765341.831507:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023741851392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765341.831553:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765341.831604:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765341/real 1775765341] req@ffff90c2034bca80 x1862023741851008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765341.831627:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023741851008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765341.831641:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765341.831646:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765341.831651:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741851008:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765341.831662:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765341.831688:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765341.831701:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765341/real 1775765341] req@ffff90c2034bea00 x1862023741851136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765341.831715:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023741851136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765341.831725:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765341.831729:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765341.831734:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765341.831762:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765341.831781:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765341.831791:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765341/real 1775765341] req@ffff90c2034bf480 x1862023741851264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765341.831804:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023741851264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765341.831813:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765341.831816:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765341.831820:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765341.831826:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851392:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765341.831844:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765341.831853:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765341/real 1775765341] req@ffff90c2034be300 x1862023741851392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765341.831866:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023741851392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765366 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765341.831935:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765341.831942:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765341.831949:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851392:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765342.855125:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765343.879135:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765344.903131:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765345.927124:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765346.951109:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765346.951121:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765346.951127:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765346.951155:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1235 00000100:00080000:1.0:1775765346.951168:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765346.951175:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765346.951202:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765346.951214:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741851520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765346.951266:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765346.951270:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765346.951273:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765346.951277:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1235 00000100:00080000:1.0:1775765346.951281:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765346.951286:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765346.951298:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765346.951315:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741851520:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765346.951317:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741851648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765346.951336:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765346.951340:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765346.951343:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765346.951347:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1235 00000100:00080000:1.0:1775765346.951350:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765346.951353:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765346.951359:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765346.951372:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741851776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765346.951383:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765346.951386:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765346.951388:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765346.951392:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1235 00000100:00080000:1.0:1775765346.951395:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765346.951398:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765346.951402:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775765346.951415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765346.951417:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741851904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765346.951471:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765346/real 1775765346] req@ffff90c203497100 x1862023741851520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765346.951493:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741851520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765346.951505:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765346.951510:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765346.951515:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741851520:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765346.951573:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851648:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765346.951606:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765346.951621:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765346/real 1775765346] req@ffff90c203495f80 x1862023741851648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765346.951637:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741851648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765346.951648:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765346.951652:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765346.951657:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765346.951663:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851776:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765346.951682:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765346.951692:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765346/real 1775765346] req@ffff90c203494000 x1862023741851776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765346.951715:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741851776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765346.951724:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765346.951728:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765346.951731:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765346.951738:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851904:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765346.951770:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765346.951776:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765346/real 1775765346] req@ffff90c203497480 x1862023741851904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765346.951784:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741851904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765371 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765346.951792:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765346.951795:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765346.951797:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741851904:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765347.975057:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765348.999119:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765350.023089:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765351.047070:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765352.071072:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765352.071085:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765352.071093:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775765352.071116:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765352.071123:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1240 00000100:00080000:0.0:1775765352.071140:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765352.071149:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765352.071221:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741852032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765352.071260:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765352.071265:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765352.071268:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765352.071273:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1240 00000100:00080000:0.0:1775765352.071277:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765352.071282:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765352.071296:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765352.071313:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741852160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765352.071322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741852032:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765352.071341:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765352.071347:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765352.071349:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765352.071354:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1240 00000100:00080000:0.0:1775765352.071358:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765352.071363:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765352.071369:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765352.071385:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741852288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765352.071396:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765352.071400:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765352.071402:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765352.071409:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1240 00000100:00080000:0.0:1775765352.071414:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765352.071417:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765352.071423:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765352.071437:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741852416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765352.071451:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765352.071510:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765352/real 1775765352] req@ffff90c203638380 x1862023741852032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765352.071535:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741852032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765352.071548:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765352.071555:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765352.071560:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741852032:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765352.071595:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765352.071637:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765352.071664:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765352/real 1775765352] req@ffff90c20363aa00 x1862023741852160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765352.071690:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741852160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765352.071708:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765352.071712:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765352.071717:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765352.071725:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765352.071753:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765352.071768:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765352/real 1775765352] req@ffff90c20363a680 x1862023741852288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765352.071782:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741852288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765352.071792:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765352.071795:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765352.071800:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765352.071806:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852416:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765352.071825:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765352.071835:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765352/real 1775765352] req@ffff90c20363bb80 x1862023741852416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765352.071848:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741852416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765352.071859:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765352.071863:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765352.071866:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852416:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765353.095149:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765354.119123:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765355.143192:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765356.167020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765357.191066:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765357.191074:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765357.191078:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765357.191087:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1245 00000100:00080000:0.0:1775765357.191109:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765357.191115:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765357.191124:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765357.191151:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741852544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765357.191173:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765357.191176:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765357.191177:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765357.191180:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1245 00000100:00080000:0.0:1775765357.191182:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765357.191185:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765357.191193:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765357.191204:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741852672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765357.191206:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741852544:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765357.191223:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765357.191227:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765357.191229:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765357.191233:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1245 00000100:00080000:0.0:1775765357.191237:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765357.191241:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765357.191247:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765357.191259:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741852800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765357.191268:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765357.191276:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765357.191280:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765357.191283:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765357.191288:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1245 00000100:00080000:0.0:1775765357.191292:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765357.191295:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765357.191300:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765357.191303:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765357/real 1775765357] req@ffff90c20363b100 x1862023741852544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765357.191312:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741852928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765357.191319:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741852544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765357.191329:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765357.191333:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765357.191336:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741852544:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765357.191376:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765357.191397:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765357.191422:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765357/real 1775765357] req@ffff90c203639500 x1862023741852672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765357.191435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741852672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765357.191441:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765357.191442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765357.191444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765357.191448:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765357.191459:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765357.191465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765357/real 1775765357] req@ffff90c203639180 x1862023741852800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765357.191472:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741852800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765357.191476:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765357.191478:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765357.191480:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765357.191483:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852928:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765357.191493:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765357.191497:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765357/real 1775765357] req@ffff90c203638700 x1862023741852928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765357.191504:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741852928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765357.191514:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765357.191516:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765357.191518:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741852928:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765358.215002:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765359.239132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765360.263000:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765361.287028:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765362.310952:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765362.310960:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765362.310971:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765362.310981:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1250 00000100:00080000:0.0:1775765362.310992:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765362.310998:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765362.311030:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741853056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765362.311045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765362.311053:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765362.311057:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765362.311059:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765362.311062:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1250 00000100:00080000:0.0:1775765362.311064:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765362.311067:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765362.311074:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765362.311085:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741853184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765362.311090:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741853056:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765362.311095:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765362.311098:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765362.311099:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765362.311103:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1250 00000100:00080000:0.0:1775765362.311105:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765362.311107:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765362.311110:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765362.311118:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741853312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765362.311124:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765362.311127:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765362.311128:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765362.311131:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1250 00000100:00080000:0.0:1775765362.311134:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765362.311136:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765362.311139:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765362.311146:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741853440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765362.311181:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765362.311204:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765362.311226:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853440:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765363.335120:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765364.359032:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765365.383271:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765365.383429:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775765365.383491:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765365.383518:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765365.383570:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765365.383586:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765365.383600:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775765365.383635:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765362/real 1775765365] req@ffff90c203638700 x1862023741853184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765365.383681:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741853184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765365.383704:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765365.383713:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765365.383721:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765365.383734:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765362/real 1775765365] req@ffff90c203639180 x1862023741853312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765365.383755:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741853312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765365.383816:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765365.383825:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765365.383831:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765365.383843:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765362/real 1775765365] req@ffff90c203639500 x1862023741853440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765365.383866:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741853440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765365.383921:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765365.383929:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765365.383936:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853440:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775765365.383950:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765362/real 1775765365] req@ffff90c203638000 x1862023741853056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765365.383977:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741853056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765365.383992:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765365.383998:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765365.384004:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741853056:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775765366.407207:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765367.431047:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765367.431063:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:3.0:1775765367.431063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765367.431070:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765367.431086:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1255 00000100:00080000:1.0:1775765367.431116:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765367.431145:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765367.431196:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741853568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765367.431239:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765367.431244:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765367.431247:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765367.431254:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1255 00000100:00080000:1.0:1775765367.431262:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765367.431284:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765367.431299:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765367.431322:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741853696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765367.431322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741853568:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765367.431348:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765367.431353:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765367.431356:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765367.431365:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1255 00000100:00080000:1.0:1775765367.431371:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765367.431376:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765367.431383:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765367.431415:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741853824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765367.431427:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775765367.431435:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765367.431441:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765367.431445:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765367.431453:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1255 00000100:00080000:1.0:1775765367.431467:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765367.431474:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765367.431486:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765367.431493:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765367/real 1775765367] req@ffff90c203494e00 x1862023741853568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765367.431519:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741853952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765367.431527:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741853568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765367.431556:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765367.431563:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765367.431573:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741853568:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765367.431613:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765367.431644:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765367.431661:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765367/real 1775765367] req@ffff90c203497480 x1862023741853696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765367.431680:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741853696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765367.431692:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765367.431696:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765367.431701:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765367.431709:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765367.431785:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765367.431801:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765367/real 1775765367] req@ffff90c203494000 x1862023741853824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765367.431817:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741853824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765367.431829:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765367.431833:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765367.431837:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765367.431853:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853952:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765367.431913:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765367.431927:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765367/real 1775765367] req@ffff90c203495f80 x1862023741853952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765367.431945:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741853952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765367.431959:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765367.431963:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765367.431968:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741853952:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765368.455042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765369.479075:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765370.503050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765371.527064:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765372.550934:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765372.550942:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765372.550947:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765372.550955:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1261 00000100:00080000:1.0:1775765372.550965:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765372.550971:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765372.550999:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765372.551013:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741854080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765372.551051:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765372.551055:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765372.551056:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765372.551060:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1261 00000100:00080000:1.0:1775765372.551063:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765372.551067:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765372.551078:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765372.551090:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741854080:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765372.551092:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741854208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765372.551105:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765372.551108:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765372.551109:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765372.551113:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1261 00000100:00080000:1.0:1775765372.551116:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765372.551119:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765372.551123:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765372.551132:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741854336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765372.551139:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765372.551141:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765372.551143:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765372.551146:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1261 00000100:00080000:1.0:1775765372.551149:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765372.551152:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765372.551156:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765372.551160:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765372.551168:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741854464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765372.551208:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765372/real 1775765372] req@ffff90c203497100 x1862023741854080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765372.551232:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741854080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765372.551244:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765372.551247:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765372.551251:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741854080:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765372.551272:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854208:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765372.551293:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765372.551303:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765372/real 1775765372] req@ffff90c203496d80 x1862023741854208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765372.551341:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741854208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765372.551349:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765372.551351:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765372.551354:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765372.551359:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854336:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765372.551374:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765372.551397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765372/real 1775765372] req@ffff90c203494380 x1862023741854336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765372.551405:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741854336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765372.551409:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765372.551411:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765372.551413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765372.551416:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854464:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765372.551426:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765372.551431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765372/real 1775765372] req@ffff90c203495c00 x1862023741854464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765372.551438:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741854464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765372.551449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765372.551451:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765372.551453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854464:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765373.575008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765374.598975:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765375.623056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765376.646985:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765377.671019:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765377.671034:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765377.671041:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765377.671057:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1266 00000100:00080000:1.0:1775765377.671074:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765377.671086:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765377.671145:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741854592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765377.671186:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765377.671194:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765377.671198:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765377.671207:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1266 00000100:00080000:1.0:1775765377.671219:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775765377.671220:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765377.671234:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765377.671252:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765377.671260:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741854592:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765377.671277:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741854720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765377.671305:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765377.671311:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765377.671314:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765377.671320:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1266 00000100:00080000:1.0:1775765377.671327:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765377.671334:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765377.671341:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765377.671359:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741854848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765377.671360:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775765377.671377:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765377.671383:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765377.671386:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765377.671393:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1266 00000100:00080000:1.0:1775765377.671398:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765377.671405:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765377.671413:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765377.671414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765377/real 1775765377] req@ffff90c203495880 x1862023741854592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765377.671430:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741854976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765377.671445:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741854592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765377.671464:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765377.671471:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765377.671479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741854592:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765377.671516:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765377.671554:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765377.671571:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765377/real 1775765377] req@ffff90c203497b80 x1862023741854720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765377.671587:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741854720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765377.671597:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765377.671601:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765377.671605:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765377.671613:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765377.671632:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765377.671643:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765377/real 1775765377] req@ffff90c203496300 x1862023741854848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765377.671690:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741854848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765377.671705:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765377.671710:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765377.671714:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765377.671721:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854976:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765377.671743:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765377.671755:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765377/real 1775765377] req@ffff90c203496a00 x1862023741854976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765402 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765377.671769:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741854976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765402 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765377.671781:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765377.671784:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765377.671788:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741854976:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765378.695048:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765379.719110:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765380.743157:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765381.767058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765382.791044:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765382.791061:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765382.791069:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765382.791084:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1271 00000100:00080000:3.0:1775765382.791111:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765382.791121:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765382.791160:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765382.791188:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741855104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765382.791234:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765382.791239:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765382.791242:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765382.791247:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1271 00000100:00080000:3.0:1775765382.791252:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765382.791257:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765382.791270:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765382.791287:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741855232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765382.791296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741855104:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765382.791318:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765382.791322:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765382.791324:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765382.791329:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1271 00000100:00080000:3.0:1775765382.791336:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765382.791340:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765382.791360:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765382.791378:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741855360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765382.791391:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765382.791396:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765382.791398:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765382.791407:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1271 00000800:00000100:2.0:1775765382.791409:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765382.791413:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765382.791420:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765382.791428:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765382.791445:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741855488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765382.791471:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765382/real 1775765382] req@ffff90c218da0380 x1862023741855104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765382.791499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741855104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765382.791513:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765382.791519:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765382.791524:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741855104:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765382.791553:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765382.791596:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765382.791643:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765382/real 1775765382] req@ffff90c218da3480 x1862023741855232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765382.791667:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741855232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765382.791678:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765382.791682:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765382.791686:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765382.791694:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765382.791720:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765382.791731:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765382/real 1775765382] req@ffff90c218da0700 x1862023741855360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765382.791746:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741855360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765382.791763:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765382.791767:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765382.791770:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765382.791777:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855488:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765382.791795:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765382.791805:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765382/real 1775765382] req@ffff90c218da3100 x1862023741855488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765407 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765382.791818:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741855488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765407 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765382.791841:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765382.791844:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765382.791848:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855488:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765383.815020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765384.839037:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765385.863124:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765386.887134:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765387.910970:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765387.910980:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765387.910985:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765387.911014:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1276 00000100:00080000:3.0:1775765387.911030:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765387.911040:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765387.911099:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741855616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765387.911120:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765387.911147:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765387.911153:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765387.911156:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765387.911161:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1276 00000100:00080000:3.0:1775765387.911165:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765387.911170:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765387.911183:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765387.911217:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741855744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765387.911218:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741855616:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765387.911234:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765387.911236:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765387.911238:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765387.911241:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1276 00000100:00080000:3.0:1775765387.911243:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765387.911246:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765387.911249:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765387.911257:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741855872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765387.911263:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765387.911271:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765387.911273:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765387.911278:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1276 00000100:00080000:3.0:1775765387.911289:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765387.911294:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765387.911300:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765387.911304:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765387.911322:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741856000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765387.911344:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765387/real 1775765387] req@ffff90c218da1f80 x1862023741855616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765387.911361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741855616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765387.911369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765387.911373:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765387.911376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741855616:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765387.911400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855744:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765387.911415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765387.911423:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765387/real 1775765387] req@ffff90c218da0000 x1862023741855744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765387.911431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741855744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765387.911437:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765387.911440:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765387.911442:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765387.911445:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855872:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765387.911454:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765387.911460:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765387/real 1775765387] req@ffff90c218da0a80 x1862023741855872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765387.911467:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741855872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765387.911472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765387.911474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765387.911476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741855872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765387.911479:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856000:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765387.911489:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765387.911494:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765387/real 1775765387] req@ffff90c218da3100 x1862023741856000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765387.911502:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741856000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765412 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765387.911509:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765387.911511:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765387.911513:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856000:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765388.935119:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765389.959148:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765390.982994:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765392.007139:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765393.031068:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765393.031076:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765393.031080:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765393.031089:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1281 00000100:00080000:0.0:1775765393.031099:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775765393.031101:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765393.031106:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765393.031139:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741856128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765393.031162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765393.031166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765393.031167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765393.031170:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1281 00000100:00080000:0.0:1775765393.031172:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765393.031174:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765393.031182:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765393.031190:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741856256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765393.031199:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765393.031202:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765393.031203:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765393.031208:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1281 00000100:00080000:0.0:1775765393.031211:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765393.031212:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741856128:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765393.031215:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765393.031220:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765393.031226:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741856384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765393.031232:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765393.031234:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765393.031236:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765393.031239:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1281 00000100:00080000:0.0:1775765393.031242:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765393.031243:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765393.031248:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765393.031258:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741856512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765393.031283:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765393.031321:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765393/real 1775765393] req@ffff90c20363b100 x1862023741856128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765393.031340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741856128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765393.031351:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765393.031356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765393.031360:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741856128:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765393.031384:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765393.031407:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765393.031419:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765393/real 1775765393] req@ffff90c20363bb80 x1862023741856256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765393.031433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741856256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765393.031463:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765393.031467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765393.031470:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765393.031476:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765393.031495:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765393.031538:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765393/real 1775765393] req@ffff90c20363a680 x1862023741856384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765393.031552:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741856384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765393.031561:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765393.031564:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765393.031567:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765393.031573:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856512:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765393.031593:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765393.031601:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765393/real 1775765393] req@ffff90c20363aa00 x1862023741856512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765393.031609:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741856512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765393.031616:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765393.031618:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765393.031620:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856512:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765394.055207:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765395.079014:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765396.103075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765397.127059:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765398.150979:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765398.150987:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765398.150991:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765398.150999:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1286 00000100:00080000:0.0:1775765398.151008:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765398.151016:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765398.151062:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741856640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775765398.151064:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765398.151091:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765398.151094:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765398.151096:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765398.151101:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1286 00000100:00080000:0.0:1775765398.151105:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765398.151110:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765398.151120:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765398.151121:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741856640:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765398.151140:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741856768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765398.151152:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765398.151156:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765398.151158:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765398.151161:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1286 00000100:00080000:0.0:1775765398.151165:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765398.151168:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765398.151171:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765398.151182:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741856896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765398.151187:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765398.151195:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765398.151198:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765398.151199:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765398.151202:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1286 00000100:00080000:0.0:1775765398.151206:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765398.151208:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765398.151211:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765398.151218:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741857024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765398.151225:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765398/real 1775765398] req@ffff90c203638380 x1862023741856640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765398.151243:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741856640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765398.151254:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765398.151260:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765398.151264:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741856640:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765398.151287:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765398.151308:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765398.151318:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765398/real 1775765398] req@ffff90c203638e00 x1862023741856768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765398.151326:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741856768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765398.151332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765398.151334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765398.151337:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765398.151340:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856896:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765398.151350:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765398.151357:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765398/real 1775765398] req@ffff90c20363b480 x1862023741856896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765398.151364:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741856896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765398.151369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765398.151370:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765398.151373:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741856896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765398.151396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857024:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765398.151415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765398.151426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765398/real 1775765398] req@ffff90c20363b800 x1862023741857024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765398.151440:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741857024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765398.151451:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765398.151475:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765398.151478:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857024:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765399.175057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765400.199109:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765401.223052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765402.247143:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765403.270940:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765403.270948:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765403.270953:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765403.270961:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1291 00000100:00080000:0.0:1775765403.270971:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765403.270977:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765403.271023:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741857152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765403.271051:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765403.271054:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765403.271056:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765403.271059:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1291 00000100:00080000:0.0:1775765403.271062:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765403.271064:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765403.271073:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765403.271084:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741857280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765403.271095:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775765403.271097:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741857152:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765403.271098:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765403.271102:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765403.271111:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1291 00010000:00010000:3.0:1775765403.271112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765403.271115:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765403.271122:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765403.271127:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765403.271157:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741857408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765403.271168:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775765403.271174:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765403.271175:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765403.271178:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765403.271184:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1291 00000100:00080000:0.0:1775765403.271190:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765403.271194:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765403.271200:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765403.271213:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765403/real 1775765403] req@ffff90c203639f80 x1862023741857152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765403.271217:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741857536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765403.271247:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741857152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765403.271255:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765403.271259:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765403.271263:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741857152:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765403.271280:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765403.271295:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765403.271308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765403/real 1775765403] req@ffff90c20363b800 x1862023741857280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765403.271316:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741857280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765403.271321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765403.271323:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765403.271326:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765403.271329:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765403.271339:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765403.271346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765403/real 1775765403] req@ffff90c20363b480 x1862023741857408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765403.271358:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741857408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765403.271377:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765403.271380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765403.271383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765403.271400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857536:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765403.271449:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765403.271458:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765403/real 1775765403] req@ffff90c203638e00 x1862023741857536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765403.271465:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741857536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765403.271472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765403.271474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765403.271476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857536:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765404.295036:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765405.319064:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765406.343132:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765407.367052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765408.390966:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765408.390976:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765408.390981:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765408.390992:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1296 00000100:00080000:0.0:1775765408.391003:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765408.391012:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765408.391025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765408.391053:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741857664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765408.391081:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765408.391085:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765408.391087:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765408.391093:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1296 00000100:00080000:0.0:1775765408.391097:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765408.391101:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765408.391111:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765408.391112:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741857664:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765408.391130:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741857792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765408.391144:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765408.391147:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765408.391150:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765408.391154:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1296 00000100:00080000:0.0:1775765408.391162:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765408.391167:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765408.391173:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1775765408.391178:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765408.391188:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741857920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765408.391200:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765408.391204:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765408.391206:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775765408.391211:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765408/real 1775765408] req@ffff90c203638380 x1862023741857664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765408.391213:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1296 00000100:00080000:0.0:1775765408.391220:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765408.391227:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765408.391232:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765408.391241:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741857664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765408.391247:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741858048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765408.391268:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765408.391274:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765408.391279:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741857664:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765408.391305:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765408.391334:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765408.391384:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765408/real 1775765408] req@ffff90c20363aa00 x1862023741857792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765408.391400:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741857792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765408.391406:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765408.391415:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765408.391417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765408.391421:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765408.391442:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765408.391450:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765408/real 1775765408] req@ffff90c20363a680 x1862023741857920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765408.391462:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741857920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765408.391468:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765408.391470:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765408.391472:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741857920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765408.391476:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858048:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765408.391495:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765408.391502:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765408/real 1775765408] req@ffff90c20363bb80 x1862023741858048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765408.391514:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741858048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765408.391525:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765408.391527:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765408.391529:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858048:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765409.415022:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765410.439045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765411.463109:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765412.486993:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765413.510970:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765413.510979:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765413.510983:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765413.510990:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1301 00000100:00080000:0.0:1775765413.511001:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765413.511008:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765413.511045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765413.511047:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741858176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765413.511089:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765413.511094:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765413.511096:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765413.511099:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1301 00000100:00080000:0.0:1775765413.511103:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765413.511106:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765413.511114:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765413.511148:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741858304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765413.511158:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775765413.511159:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741858176:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765413.511171:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765413.511172:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765413.511178:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1301 00000100:00080000:0.0:1775765413.511182:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765413.511185:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765413.511189:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765413.511199:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741858432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765413.511205:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765413.511208:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765413.511209:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765413.511211:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1301 00000100:00080000:0.0:1775765413.511214:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765413.511215:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765413.511219:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:1.0:1775765413.511224:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765413.511228:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741858560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765413.511266:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765413/real 1775765413] req@ffff90c20363b100 x1862023741858176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765413.511285:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741858176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765413.511295:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765413.511335:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765413.511339:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741858176:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765413.511362:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858304:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765413.511386:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765413.511409:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765413/real 1775765413] req@ffff90c203638000 x1862023741858304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765413.511422:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741858304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765413.511431:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765413.511434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765413.511438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765413.511444:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858432:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765413.511461:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765413.511469:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765413/real 1775765413] req@ffff90c203639500 x1862023741858432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765413.511476:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741858432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765413.511482:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765413.511484:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765413.511486:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765413.511490:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858560:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765413.511500:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765413.511505:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765413/real 1775765413] req@ffff90c203639180 x1862023741858560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765413.511512:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741858560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765413.511518:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765413.511520:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765413.511522:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858560:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765414.535044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765415.559046:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765416.583008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765417.607077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765418.631001:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765418.631011:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765418.631017:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765418.631029:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1307 00000100:00080000:1.0:1775765418.631042:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775765418.631046:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765418.631050:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765418.631096:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741858688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765418.631124:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765418.631127:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765418.631128:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765418.631131:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1307 00000100:00080000:1.0:1775765418.631134:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765418.631136:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765418.631146:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765418.631155:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741858816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765418.631163:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765418.631168:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765418.631169:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741858688:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765418.631170:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765418.631178:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1307 00000100:00080000:1.0:1775765418.631181:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765418.631184:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765418.631186:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765418.631194:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741858944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765418.631204:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765418.631207:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765418.631209:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765418.631212:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1307 00000100:00080000:1.0:1775765418.631214:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765418.631216:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765418.631219:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765418.631227:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741859072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765418.631256:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765418.631333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765418/real 1775765418] req@ffff90c203496680 x1862023741858688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765418.631355:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741858688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765418.631368:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765418.631373:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765418.631378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741858688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765418.631405:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765418.631428:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765418.631441:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765418/real 1775765418] req@ffff90c203494a80 x1862023741858816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765418.631455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741858816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765418.631465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765418.631468:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765418.631471:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765418.631478:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765418.631497:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765418.631507:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765418/real 1775765418] req@ffff90c203495180 x1862023741858944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765418.631520:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741858944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765418.631529:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765418.631533:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765418.631537:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741858944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765418.631544:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859072:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765418.631564:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765418.631574:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765418/real 1775765418] req@ffff90c203495500 x1862023741859072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765443 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765418.631585:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741859072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765443 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765418.631597:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765418.631601:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765418.631604:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859072:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765419.655048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765420.679045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765421.703075:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765422.727039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765423.750994:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765423.751002:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765423.751007:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765423.751015:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1312 00000100:00080000:0.0:1775765423.751025:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765423.751032:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765423.751049:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765423.751067:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741859200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765423.751090:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765423.751093:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765423.751094:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765423.751097:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1312 00000100:00080000:0.0:1775765423.751100:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765423.751102:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765423.751111:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765423.751122:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741859328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765423.751139:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741859200:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765423.751141:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765423.751154:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765423.751155:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765423.751160:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1312 00000100:00080000:0.0:1775765423.751162:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765423.751165:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765423.751169:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765423.751183:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741859456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765423.751190:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765423.751192:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765423.751194:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765423.751196:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1312 00000100:00080000:0.0:1775765423.751198:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765423.751201:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765423.751204:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765423.751224:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741859584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765423.751255:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765423.751304:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765423/real 1775765423] req@ffff90c203638700 x1862023741859200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765423.751322:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741859200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765423.751332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765423.751335:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765423.751339:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741859200:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765423.751363:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765423.751391:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765423.751401:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765423/real 1775765423] req@ffff90c203639180 x1862023741859328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765423.751410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741859328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765423.751416:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765423.751418:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765423.751420:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765423.751424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765423.751435:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765423.751440:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765423/real 1775765423] req@ffff90c203639500 x1862023741859456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765423.751448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741859456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765423.751453:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765423.751454:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765423.751456:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765423.751460:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859584:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765423.751470:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765423.751475:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765423/real 1775765423] req@ffff90c203638000 x1862023741859584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765448 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765423.751481:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741859584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765448 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765423.751493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765423.751496:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765423.751498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859584:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765424.775089:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765425.799037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765426.823004:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765427.847041:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765428.871012:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765428.871024:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765428.871030:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765428.871042:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1317 00000100:00080000:0.0:1775765428.871057:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765428.871068:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765428.871125:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765428.871133:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741859712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765428.871186:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765428.871190:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765428.871192:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765428.871196:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1317 00000100:00080000:0.0:1775765428.871199:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765428.871203:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765428.871215:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765428.871228:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741859840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765428.871235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741859712:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765428.871239:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765428.871243:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765428.871245:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765428.871248:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1317 00000100:00080000:0.0:1775765428.871251:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765428.871254:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765428.871258:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765428.871267:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741859968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765428.871274:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765428.871276:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765428.871277:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765428.871281:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1317 00000100:00080000:0.0:1775765428.871285:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765428.871288:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765428.871292:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765428.871301:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741860096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765428.871360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765428.871390:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765428.871405:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860096:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765429.895263:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765430.919369:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765431.943155:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775765431.943247:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:2.0:1775765431.943303:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765431.943331:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:2.0:1775765431.943388:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:2.0:1775765431.943401:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:2.0:1775765431.943411:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:1.0:1775765431.943524:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765428/real 1775765431] req@ffff90c20363bb80 x1862023741859840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765431.943559:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741859840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765431.943573:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765431.943580:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765431.943584:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765431.943594:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765428/real 1775765431] req@ffff90c20363a680 x1862023741859968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765431.943608:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741859968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765431.943617:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765431.943620:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765431.943624:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741859968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765431.943632:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765428/real 1775765431] req@ffff90c20363aa00 x1862023741860096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765431.943644:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741860096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765431.943654:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765431.943658:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765431.943661:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860096:192.168.202.155@tcp:8:kworker.0 00000100:00100000:1.0:1775765431.943668:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765428/real 1775765431] req@ffff90c20363b100 x1862023741859712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765431.943681:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741859712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765453 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765431.943688:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765431.943691:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765431.943694:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741859712:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775765432.967206:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765433.991049:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765433.991068:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765433.991076:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765433.991095:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1322 00000100:00080000:0.0:1775765433.991130:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765433.991142:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765433.991185:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765433.991209:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741860224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765433.991259:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765433.991264:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765433.991267:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765433.991272:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1322 00000100:00080000:0.0:1775765433.991278:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765433.991282:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765433.991296:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765433.991316:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741860352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765433.991330:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1775765433.991334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741860224:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765433.991335:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765433.991339:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765433.991352:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1322 00000100:00080000:0.0:1775765433.991355:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765433.991359:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765433.991364:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765433.991384:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741860480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765433.991411:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765433.991417:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765433.991421:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765433.991431:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1322 00000100:00080000:0.0:1775765433.991455:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765433.991463:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765433.991472:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:1.0:1775765433.991482:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765433.991498:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741860608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765433.991556:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765433/real 1775765433] req@ffff90c203638380 x1862023741860224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765433.991608:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741860224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765433.991628:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765433.991636:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765433.991644:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741860224:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765433.991705:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765433.991747:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765433.991769:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765433/real 1775765433] req@ffff90c203638e00 x1862023741860352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765433.991791:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741860352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765433.991805:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765433.991811:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765433.991816:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765433.991827:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765433.991862:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765433.991881:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765433/real 1775765433] req@ffff90c20363b480 x1862023741860480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765433.991949:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741860480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765433.991973:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765433.991979:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765433.991985:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765433.991996:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860608:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765433.992036:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765433.992053:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765433/real 1775765433] req@ffff90c20363b800 x1862023741860608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765433.992072:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741860608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765458 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765433.992090:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765433.992096:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765433.992102:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860608:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765435.015214:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765436.039120:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765437.063208:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765438.087134:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765439.111041:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765439.111053:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765439.111060:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765439.111085:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1327 00000100:00080000:3.0:1775765439.111100:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765439.111109:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765439.111112:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765439.111159:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741860736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765439.111188:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765439.111192:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765439.111194:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765439.111200:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1327 00000100:00080000:3.0:1775765439.111206:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765439.111211:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765439.111216:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741860736:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775765439.111224:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765439.111242:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741860864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765439.111252:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765439.111255:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765439.111257:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765439.111260:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1327 00000100:00080000:3.0:1775765439.111264:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765439.111266:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765439.111272:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765439.111281:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741860992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765439.111288:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1775765439.111292:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765439.111293:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765439.111296:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765439.111300:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1327 00000100:00080000:3.0:1775765439.111303:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765439.111306:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765439.111310:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765439.111320:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741861120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775765439.111358:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765439/real 1775765439] req@ffff90c218da0700 x1862023741860736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765439.111378:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741860736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765439.111389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765439.111394:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765439.111398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741860736:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765439.111441:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860864:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765439.111457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765439.111465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765439/real 1775765439] req@ffff90c218da3480 x1862023741860864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765439.111474:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741860864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765439.111480:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765439.111482:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765439.111484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765439.111488:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860992:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775765439.111515:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765439.111520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765439/real 1775765439] req@ffff90c218da0380 x1862023741860992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765439.111527:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741860992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765439.111533:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765439.111534:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765439.111536:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741860992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775765439.111540:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861120:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775765439.111551:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765439.111556:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765439/real 1775765439] req@ffff90c218da2d80 x1862023741861120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775765439.111564:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741861120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765439.111571:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775765439.111573:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775765439.111574:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861120:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765440.134990:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765441.159041:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765442.182980:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765443.207139:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765444.231089:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765444.231105:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765444.231112:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765444.231126:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1332 00000100:00080000:1.0:1775765444.231140:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765444.231148:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765444.231194:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765444.231197:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741861248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765444.231236:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765444.231241:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765444.231243:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765444.231247:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1332 00000100:00080000:1.0:1775765444.231251:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765444.231256:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765444.231268:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765444.231285:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741861376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765444.231292:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741861248:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765444.231311:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765444.231314:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765444.231316:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765444.231321:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1332 00000100:00080000:1.0:1775765444.231326:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765444.231330:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765444.231335:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765444.231351:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741861504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765444.231361:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765444.231364:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765444.231367:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765444.231371:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1332 00000100:00080000:1.0:1775765444.231374:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765444.231379:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765444.231387:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775765444.231390:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765444.231407:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741861632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765444.231440:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765444/real 1775765444] req@ffff90c203494700 x1862023741861248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765444.231462:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741861248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765444.231474:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765444.231479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765444.231484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741861248:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765444.231510:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765444.231532:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765444.231544:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765444/real 1775765444] req@ffff90c203495500 x1862023741861376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765444.231557:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741861376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765444.231565:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765444.231569:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765444.231573:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765444.231578:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765444.231594:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765444.231603:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765444/real 1775765444] req@ffff90c203495180 x1862023741861504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765444.231614:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741861504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765444.231622:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765444.231625:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765444.231628:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765444.231634:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861632:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765444.231649:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765444.231658:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765444/real 1775765444] req@ffff90c203494a80 x1862023741861632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765444.231669:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741861632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765444.231679:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765444.231682:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765444.231685:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861632:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765445.255204:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765446.279113:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765447.303091:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765448.327082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765449.351038:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765449.351056:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765449.351063:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765449.351081:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1337 00000100:00080000:0.0:1775765449.351098:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765449.351109:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765449.351162:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741861760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775765449.351165:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765449.351212:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765449.351218:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765449.351221:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765449.351226:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1337 00000100:00080000:0.0:1775765449.351232:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765449.351237:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765449.351249:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765449.351271:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741861760:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765449.351272:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741861888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765449.351313:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765449.351318:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765449.351321:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765449.351326:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1337 00000100:00080000:0.0:1775765449.351331:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765449.351335:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765449.351343:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765449.351357:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741862016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765449.351370:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765449.351374:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765449.351379:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765449.351387:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765449.351392:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1337 00000100:00080000:0.0:1775765449.351400:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765449.351405:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765449.351412:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765449.351429:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741862144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765449.351469:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765449/real 1775765449] req@ffff90c203639f80 x1862023741861760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765449.351518:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741861760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765449.351540:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765449.351545:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765449.351550:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741861760:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765449.351581:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765449.351615:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765449.351638:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765449/real 1775765449] req@ffff90c20363b800 x1862023741861888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765449.351676:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741861888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765449.351693:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765449.351700:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765449.351707:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741861888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765449.351720:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765449.351760:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765449.351782:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765449/real 1775765449] req@ffff90c20363b480 x1862023741862016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765449.351806:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741862016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765449.351823:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765449.351830:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765449.351837:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765449.351850:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862144:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765449.351923:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765449.351939:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765449/real 1775765449] req@ffff90c203638e00 x1862023741862144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765449.351954:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741862144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765449.352011:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765449.352018:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765449.352024:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862144:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765450.377158:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765451.399194:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765452.423115:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765453.447052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765454.470982:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765454.470990:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765454.470995:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765454.471004:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1342 00000100:00080000:0.0:1775765454.471014:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765454.471054:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765454.471072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765454.471091:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741862272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765454.471116:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765454.471119:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765454.471120:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765454.471124:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1342 00000100:00080000:0.0:1775765454.471127:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765454.471129:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765454.471138:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765454.471147:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741862400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765454.471156:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765454.471159:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765454.471161:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765454.471166:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1342 00000100:00080000:0.0:1775765454.471169:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765454.471171:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741862272:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765454.471174:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765454.471183:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765454.471192:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741862528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765454.471198:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765454.471201:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765454.471203:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765454.471206:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1342 00000100:00080000:0.0:1775765454.471211:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765454.471214:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765454.471217:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765454.471226:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741862656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765454.471285:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765454.471329:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765454/real 1775765454] req@ffff90c203638380 x1862023741862272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765454.471346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741862272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765454.471356:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765454.471359:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765454.471362:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741862272:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765454.471388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862400:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765454.471404:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765454.471412:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765454/real 1775765454] req@ffff90c20363b100 x1862023741862400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765454.471420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741862400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765454.471426:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765454.471428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765454.471430:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765454.471435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862528:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765454.471446:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765454.471453:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765454/real 1775765454] req@ffff90c20363aa00 x1862023741862528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765454.471461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741862528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765454.471466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765454.471467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765454.471469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765454.471473:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862656:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765454.471484:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765454.471489:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765454/real 1775765454] req@ffff90c20363a680 x1862023741862656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765454.471495:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741862656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765454.471502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765454.471505:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765454.471507:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862656:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765455.495052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765456.519103:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765457.543125:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765458.567172:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765459.591051:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765459.591059:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765459.591064:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765459.591074:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1348 00000100:00080000:3.0:1775765459.591084:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765459.591091:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765459.591123:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765459.591127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741862784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765459.591151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765459.591154:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765459.591156:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765459.591158:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1348 00000100:00080000:3.0:1775765459.591160:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765459.591163:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765459.591172:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765459.591182:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741862912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765459.591190:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765459.591193:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765459.591194:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765459.591203:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1348 00000100:00100000:2.0:1775765459.591205:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741862784:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765459.591207:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765459.591218:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765459.591226:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765459.591242:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741863040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765459.591254:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765459.591257:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765459.591258:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765459.591261:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1348 00000100:00080000:3.0:1775765459.591273:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765459.591276:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765459.591279:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765459.591289:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741863168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765459.591332:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765459.591387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765459/real 1775765459] req@ffff90c218da0e00 x1862023741862784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765459.591414:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741862784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765459.591429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765459.591435:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765459.591440:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741862784:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765459.591470:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765459.591494:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765459.591507:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765459/real 1775765459] req@ffff90c218da1880 x1862023741862912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765459.591521:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741862912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765459.591531:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765459.591535:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765459.591538:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741862912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765459.591544:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765459.591561:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765459.591570:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765459/real 1775765459] req@ffff90c218da3800 x1862023741863040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765459.591583:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741863040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765459.591600:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765459.591603:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765459.591607:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765459.591613:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863168:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765459.591630:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765459.591639:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765459/real 1775765459] req@ffff90c218da2d80 x1862023741863168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765459.591650:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741863168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765459.591661:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765459.591665:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765459.591669:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863168:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765460.615096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765461.639099:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765462.663054:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765463.687055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765464.710998:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765464.711012:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765464.711018:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765464.711033:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1353 00000100:00080000:3.0:1775765464.711049:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765464.711058:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765464.711067:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765464.711107:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741863296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765464.711133:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765464.711136:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765464.711138:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765464.711141:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1353 00000100:00080000:3.0:1775765464.711144:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765464.711148:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765464.711159:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765464.711171:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741863424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765464.711173:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741863296:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765464.711187:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765464.711190:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765464.711191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765464.711193:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1353 00000100:00080000:3.0:1775765464.711195:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765464.711197:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765464.711200:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765464.711208:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741863552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765464.711214:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765464.711216:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765464.711217:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765464.711220:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1353 00000100:00080000:3.0:1775765464.711244:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765464.711247:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765464.711250:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765464.711256:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765464.711262:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741863680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765464.711301:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765464/real 1775765464] req@ffff90c218da0380 x1862023741863296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765464.711318:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741863296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765464.711327:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765464.711331:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765464.711334:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741863296:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765464.711358:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863424:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765464.711372:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765464.711380:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765464/real 1775765464] req@ffff90c218da3480 x1862023741863424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765464.711388:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741863424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765464.711393:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765464.711396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765464.711398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765464.711401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863552:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765464.711411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765464.711416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765464/real 1775765464] req@ffff90c218da0700 x1862023741863552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765464.711423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741863552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765464.711428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765464.711430:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765464.711432:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765464.711435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863680:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765464.711445:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765464.711450:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765464/real 1775765464] req@ffff90c218da3100 x1862023741863680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765489 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765464.711458:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741863680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765489 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765464.711465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765464.711467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765464.711469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863680:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765465.735096:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765466.759104:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765467.783198:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765468.807156:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765469.831079:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765469.831093:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765469.831101:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765469.831114:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1358 00010000:00010000:0.0:1775765469.831126:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765469.831129:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765469.831144:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765469.831208:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741863808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765469.831250:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765469.831255:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765469.831258:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765469.831263:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1358 00000100:00080000:1.0:1775765469.831267:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765469.831272:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765469.831285:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765469.831302:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741863936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765469.831320:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765469.831334:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765469.831335:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741863808:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765469.831337:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765469.831359:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1358 00000100:00080000:1.0:1775765469.831364:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765469.831368:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765469.831374:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765469.831390:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741864064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765469.831400:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765469.831405:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765469.831407:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765469.831412:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1358 00000100:00080000:1.0:1775765469.831426:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765469.831430:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765469.831437:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765469.831454:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741864192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765469.831492:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765469.831557:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765469/real 1775765469] req@ffff90c203496680 x1862023741863808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765469.831581:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741863808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765469.831593:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765469.831598:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765469.831603:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741863808:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765469.831633:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863936:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765469.831657:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765469.831669:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765469/real 1775765469] req@ffff90c203496a00 x1862023741863936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765469.831680:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741863936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765469.831707:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765469.831711:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765469.831714:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741863936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765469.831720:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765469.831735:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765469.831744:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765469/real 1775765469] req@ffff90c203496300 x1862023741864064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765469.831755:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741864064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765469.831761:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765469.831764:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765469.831766:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765469.831791:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864192:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765469.831812:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765469.831825:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765469/real 1775765469] req@ffff90c203497b80 x1862023741864192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765469.831841:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741864192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765494 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765469.831856:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765469.831861:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765469.831867:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864192:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765470.855154:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765471.879079:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765472.903150:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765473.927049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765474.950987:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765474.951002:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765474.951011:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765474.951025:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1363 00000100:00080000:1.0:1775765474.951041:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765474.951053:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765474.951108:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741864320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765474.951147:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765474.951156:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765474.951161:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765474.951171:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1363 00010000:00010000:0.0:1775765474.951179:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765474.951181:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765474.951191:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765474.951209:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765474.951211:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741864320:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765474.951242:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741864448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765474.951265:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765474.951272:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765474.951275:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765474.951283:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1363 00000100:00080000:1.0:1775765474.951297:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765474.951304:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765474.951312:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765474.951336:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741864576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765474.951342:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775765474.951404:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765474.951413:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765474.951417:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765474.951427:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765474/real 1775765474] req@ffff90c203495880 x1862023741864320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775765474.951438:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1363 00000100:00080000:1.0:1775765474.951467:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765474.951479:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765474.951482:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741864320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1775765474.951511:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775765474.951513:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765474.951534:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765474.951552:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741864320:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765474.951573:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741864704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765474.951592:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765474.951639:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765474.951662:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765474/real 1775765474] req@ffff90c203495c00 x1862023741864448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765474.951679:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741864448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765474.951690:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765474.951695:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765474.951699:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765474.951707:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765474.951786:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765474.951810:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765474/real 1775765474] req@ffff90c203494380 x1862023741864576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765474.951837:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741864576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765474.951854:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765474.951861:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765474.951868:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765474.951932:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864704:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765474.951965:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765474.951980:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765474/real 1775765474] req@ffff90c203496d80 x1862023741864704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765474.951996:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741864704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765499 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765474.952010:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765474.952014:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765474.952018:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864704:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765475.975132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765476.999231:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765478.023162:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765479.047082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765480.071008:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765480.071017:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765480.071022:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765480.071032:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1368 00000100:00080000:1.0:1775765480.071058:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765480.071066:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765480.071108:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741864832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775765480.071108:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765480.071137:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765480.071141:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765480.071143:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765480.071148:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1368 00000100:00080000:1.0:1775765480.071151:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765480.071154:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765480.071162:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765480.071175:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741864960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765480.071184:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765480.071187:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765480.071189:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765480.071195:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1368 00000100:00100000:2.0:1775765480.071198:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741864832:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765480.071199:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765480.071210:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765480.071217:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765480.071226:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741865088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765480.071234:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765480.071236:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765480.071238:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765480.071241:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1368 00000100:00080000:1.0:1775765480.071243:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765480.071245:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765480.071248:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765480.071256:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741865216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765480.071330:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765480.071383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765480/real 1775765480] req@ffff90c203497100 x1862023741864832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765480.071415:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741864832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765480.071428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765480.071433:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765480.071437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741864832:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765480.071464:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864960:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765480.071483:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765480.071493:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765480/real 1775765480] req@ffff90c203495f80 x1862023741864960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765480.071504:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741864960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765480.071511:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765480.071515:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765480.071517:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741864960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765480.071522:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865088:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765480.071535:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765480.071542:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765480/real 1775765480] req@ffff90c203494000 x1862023741865088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765480.071552:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741865088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765480.071559:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765480.071561:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765480.071563:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765480.071569:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865216:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765480.071583:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765480.071590:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765480/real 1775765480] req@ffff90c203497480 x1862023741865216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765480.071599:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741865216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765480.071609:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765480.071611:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765480.071614:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865216:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765481.095133:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765482.119132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765483.143156:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765484.167151:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765485.191012:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765485.191019:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765485.191024:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765485.191033:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1373 00000100:00080000:0.0:1775765485.191043:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765485.191059:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765485.191069:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765485.191096:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741865344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765485.191117:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765485.191120:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765485.191121:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765485.191124:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1373 00000100:00080000:0.0:1775765485.191127:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765485.191129:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765485.191137:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765485.191147:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741865472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765485.191162:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765485.191164:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741865344:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765485.191166:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765485.191170:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765485.191179:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1373 00000100:00080000:0.0:1775765485.191182:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765485.191184:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765485.191187:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765485.191195:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741865600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765485.191202:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765485.191204:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765485.191206:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765485.191208:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1373 00000100:00080000:0.0:1775765485.191213:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765485.191216:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765485.191219:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765485.191226:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741865728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765485.191255:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765485.191297:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765485/real 1775765485] req@ffff90c20363bb80 x1862023741865344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765485.191314:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741865344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765485.191324:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765485.191328:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765485.191331:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741865344:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765485.191354:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765485.191369:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765485.191376:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765485/real 1775765485] req@ffff90c203638000 x1862023741865472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765485.191386:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741865472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765485.191392:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765485.191394:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765485.191397:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765485.191401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765485.191410:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765485.191416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765485/real 1775765485] req@ffff90c203639500 x1862023741865600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765485.191424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741865600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765485.191429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765485.191431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765485.191433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765485.191436:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865728:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765485.191447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765485.191452:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765485/real 1775765485] req@ffff90c203639180 x1862023741865728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765485.191459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741865728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765485.191471:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765485.191474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765485.191476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865728:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765486.215070:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765487.239151:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765488.263125:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765489.287111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765490.311040:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765490.311068:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765490.311074:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765490.311086:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1378 00000100:00080000:0.0:1775765490.311104:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765490.311113:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765490.311144:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765490.311191:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741865856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765490.311228:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765490.311232:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765490.311234:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765490.311239:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1378 00000100:00080000:0.0:1775765490.311244:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765490.311274:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765490.311291:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765490.311324:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741865856:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765490.311335:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741865984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765490.311353:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765490.311358:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765490.311361:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765490.311369:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1378 00000100:00080000:0.0:1775765490.311392:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765490.311400:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765490.311411:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765490.311439:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765490.311442:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741866112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765490.311466:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765490.311496:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765490.311499:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765490.311512:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765490/real 1775765490] req@ffff90c203638700 x1862023741865856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765490.311513:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1378 00000100:00080000:0.0:1775765490.311538:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765490.311568:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765490.311598:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765490.311619:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741865856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765490.311636:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741866240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765490.311645:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765490.311662:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765490.311671:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741865856:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765490.311708:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765490.311738:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765490.311755:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765490/real 1775765490] req@ffff90c203639180 x1862023741865984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765490.311775:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741865984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765490.311789:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765490.311795:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765490.311801:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741865984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765490.311813:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765490.311852:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765490.311865:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765490/real 1775765490] req@ffff90c203639500 x1862023741866112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765490.311877:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741866112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765490.311926:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765490.311930:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765490.311933:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765490.311940:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866240:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765490.311962:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765490.311972:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765490/real 1775765490] req@ffff90c203638000 x1862023741866240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765490.311983:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741866240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765490.311994:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765490.311998:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765490.312000:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866240:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765491.335070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765492.359085:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765493.383148:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765494.407117:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765495.431060:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765495.431071:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765495.431077:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765495.431090:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1383 00000100:00080000:0.0:1775765495.431102:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775765495.431103:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765495.431109:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765495.431146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741866368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765495.431170:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765495.431173:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765495.431174:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765495.431178:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1383 00000100:00080000:0.0:1775765495.431180:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765495.431183:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765495.431191:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765495.431202:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741866496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765495.431210:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741866368:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765495.431215:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765495.431218:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765495.431220:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765495.431224:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1383 00000100:00080000:0.0:1775765495.431226:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765495.431229:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765495.431233:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765495.431242:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741866624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765495.431249:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765495.431252:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765495.431253:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765495.431256:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1383 00000100:00080000:0.0:1775765495.431259:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765495.431262:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765495.431266:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765495.431274:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741866752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765495.431314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765495.431359:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765495.431393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866752:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765496.455136:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765497.479110:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765498.503066:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765498.503197:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775765498.503251:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765498.503275:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765498.503322:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765498.503334:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765498.503351:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775765498.503390:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765495/real 1775765498] req@ffff90c20363a680 x1862023741866496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765498.503431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741866496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765498.503449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765498.503457:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765498.503463:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765498.503475:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765495/real 1775765498] req@ffff90c20363aa00 x1862023741866624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765498.503528:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741866624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765498.503539:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765498.503544:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765498.503548:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765498.503558:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765495/real 1775765498] req@ffff90c20363b100 x1862023741866752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765498.503572:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741866752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765498.503598:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765498.503603:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765498.503609:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741866752:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775765498.503618:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765495/real 1775765498] req@ffff90c20363bb80 x1862023741866368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765498.503635:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741866368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765498.503645:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765498.503650:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765498.503654:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741866368:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775765499.527123:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765500.551013:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765500.551025:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765500.551032:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765500.551047:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1389 00000100:00080000:0.0:1775765500.551066:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765500.551078:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765500.551149:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741866880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765500.551162:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765500.551178:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765500.551184:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765500.551186:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765500.551193:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1389 00000100:00080000:0.0:1775765500.551197:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765500.551203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765500.551213:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765500.551224:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741866880:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765500.551230:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741867008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765500.551241:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765500.551244:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765500.551245:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765500.551248:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1389 00000100:00080000:0.0:1775765500.551251:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765500.551253:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765500.551257:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765500.551265:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741867136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765500.551272:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765500.551277:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765500.551278:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765500.551282:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1389 00000100:00080000:0.0:1775765500.551285:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765500.551288:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765500.551292:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765500.551300:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741867264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765500.551318:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765500.551373:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765500/real 1775765500] req@ffff90c203638380 x1862023741866880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765500.551402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741866880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765500.551417:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765500.551423:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765500.551428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741866880:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765500.551461:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765500.551525:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765500.551543:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765500/real 1775765500] req@ffff90c203638e00 x1862023741867008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765500.551559:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741867008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765500.551570:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765500.551574:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765500.551578:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765500.551586:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765500.551612:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765500.551625:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765500/real 1775765500] req@ffff90c20363b480 x1862023741867136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765500.551640:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741867136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765500.551651:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765500.551656:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765500.551660:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765500.551669:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867264:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765500.551697:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765500.551710:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765500/real 1775765500] req@ffff90c20363b800 x1862023741867264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765500.551733:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741867264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765500.551748:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765500.551752:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765500.551756:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867264:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765501.575128:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765502.599108:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765503.623132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765504.647085:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765505.671085:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765505.671099:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765505.671108:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765505.671125:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1394 00010000:00010000:3.0:1775765505.671141:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765505.671143:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765505.671164:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765505.671250:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741867392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765505.671299:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765505.671306:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765505.671308:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765505.671314:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1394 00000100:00080000:0.0:1775765505.671321:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765505.671327:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765505.671340:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765505.671372:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741867520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765505.671393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741867392:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765505.671397:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765505.671423:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765505.671452:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765505.671462:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1394 00000100:00080000:0.0:1775765505.671467:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765505.671473:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765505.671481:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765505.671500:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741867648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765505.671513:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765505.671518:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765505.671521:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765505.671525:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1394 00000100:00080000:0.0:1775765505.671530:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765505.671533:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765505.671539:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765505.671555:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741867776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765505.671574:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765505.671645:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765505/real 1775765505] req@ffff90c203639f80 x1862023741867392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765505.671674:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741867392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765505.671690:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765505.671696:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765505.671701:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741867392:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765505.671736:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867520:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765505.671763:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765505.671794:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765505/real 1775765505] req@ffff90c20363b800 x1862023741867520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765505.671811:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741867520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765505.671831:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765505.671835:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765505.671839:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765505.671847:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867648:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765505.671868:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765505.671880:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765505/real 1775765505] req@ffff90c20363b480 x1862023741867648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765505.671961:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741867648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765505.671977:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765505.671984:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765505.671990:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765505.672003:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867776:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765505.672038:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765505.672052:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765505/real 1775765505] req@ffff90c203638e00 x1862023741867776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765530 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765505.672067:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741867776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765530 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765505.672097:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765505.672101:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765505.672106:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741867776:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765506.695127:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765507.719101:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765508.743110:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765509.767102:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765510.791041:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765510.791051:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765510.791056:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765510.791067:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1399 00000100:00080000:0.0:1775765510.791081:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765510.791088:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765510.791130:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741867904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765510.791138:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765510.791160:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765510.791167:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765510.791170:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765510.791177:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1399 00000100:00080000:0.0:1775765510.791184:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765510.791191:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765510.791204:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765510.791211:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741867904:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765510.791249:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741868032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765510.791276:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765510.791281:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765510.791284:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765510.791289:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1399 00000100:00080000:0.0:1775765510.791296:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765510.791301:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765510.791309:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765510.791330:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741868160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765510.791335:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765510.791357:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765510.791362:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765510.791364:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765510.791372:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1399 00000100:00080000:0.0:1775765510.791400:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765510.791423:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765510.791430:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765510.791442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765510/real 1775765510] req@ffff90c203638380 x1862023741867904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765510.791447:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741868288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765510.791502:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741867904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765510.791514:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765510.791521:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765510.791531:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741867904:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765510.791571:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765510.791592:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765510.791600:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765510/real 1775765510] req@ffff90c20363bb80 x1862023741868032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765510.791609:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741868032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765510.791615:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765510.791617:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765510.791620:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765510.791624:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765510.791634:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765510.791641:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765510/real 1775765510] req@ffff90c20363b100 x1862023741868160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765510.791649:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741868160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765510.791654:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765510.791655:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765510.791657:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765510.791669:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868288:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765510.791680:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765510.791685:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765510/real 1775765510] req@ffff90c20363aa00 x1862023741868288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765535 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765510.791692:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741868288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765535 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765510.791707:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765510.791709:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765510.791711:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868288:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765511.815209:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765512.839217:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765513.863066:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765514.887156:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765515.911032:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765515.911063:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765515.911070:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765515.911087:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1404 00000100:00080000:0.0:1775765515.911105:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765515.911116:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765515.911189:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741868416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765515.911200:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765515.911245:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765515.911253:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765515.911256:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765515.911265:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1404 00000100:00080000:0.0:1775765515.911273:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765515.911282:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765515.911299:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765515.911300:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741868416:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765515.911350:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741868544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765515.911372:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765515.911379:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765515.911382:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765515.911390:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1404 00000100:00080000:0.0:1775765515.911397:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765515.911403:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765515.911413:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765515.911427:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765515.911440:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741868672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765515.911457:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765515.911463:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765515.911466:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765515.911474:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1404 00000100:00080000:0.0:1775765515.911481:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765515.911488:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765515/real 1775765515] req@ffff90c20363a680 x1862023741868416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765515.911495:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765515.911514:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765515.911526:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741868416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765515.911541:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775765515.911542:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741868800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765515.911555:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765515.911569:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741868416:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765515.911606:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868544:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765515.911630:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765515.911643:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765515/real 1775765515] req@ffff90c203638000 x1862023741868544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765515.911656:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741868544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765515.911665:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765515.911669:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765515.911672:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765515.911678:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765515.911693:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765515.911702:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765515/real 1775765515] req@ffff90c203639500 x1862023741868672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765515.911714:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741868672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765515.911723:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765515.911726:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765515.911729:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765515.911736:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868800:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765515.911752:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765515.911760:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765515/real 1775765515] req@ffff90c203639180 x1862023741868800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765515.911772:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741868800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765540 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765515.911783:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765515.911786:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765515.911789:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741868800:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765516.935064:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765517.959124:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765518.983133:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765520.007119:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765521.031037:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765521.031051:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765521.031058:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765521.031073:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1409 00000100:00080000:0.0:1775765521.031092:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775765521.031092:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765521.031102:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765521.031156:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741868928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765521.031191:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765521.031196:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765521.031198:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765521.031203:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1409 00000100:00080000:0.0:1775765521.031207:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765521.031212:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765521.031225:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765521.031240:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741868928:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765521.031246:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741869056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765521.031261:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765521.031264:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765521.031266:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765521.031271:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1409 00000100:00080000:0.0:1775765521.031275:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765521.031294:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765521.031302:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765521.031316:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741869184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765521.031327:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765521.031331:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765521.031333:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765521.031338:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1409 00000800:00000100:2.0:1775765521.031344:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765521.031347:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765521.031353:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765521.031359:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765521.031373:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741869312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765521.031402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765521/real 1775765521] req@ffff90c203638700 x1862023741868928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765521.031420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741868928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765521.031430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765521.031434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765521.031437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741868928:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765521.031460:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869056:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765521.031478:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765521.031487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765521/real 1775765521] req@ffff90c203639180 x1862023741869056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765521.031497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741869056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765521.031503:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765521.031505:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765521.031507:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765521.031510:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869184:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765521.031520:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765521.031525:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765521/real 1775765521] req@ffff90c203639500 x1862023741869184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765521.031533:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741869184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765521.031538:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765521.031540:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765521.031542:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765521.031545:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869312:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765521.031555:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765521.031560:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765521/real 1775765521] req@ffff90c203638000 x1862023741869312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765521.031566:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741869312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765521.031578:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765521.031580:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765521.031582:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869312:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765522.055160:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765523.079061:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765524.103066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765525.127082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765526.151062:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765526.151076:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765526.151082:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765526.151097:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1414 00000100:00080000:0.0:1775765526.151115:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765526.151126:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765526.151147:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765526.151195:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741869440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765526.151302:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765526.151308:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765526.151310:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765526.151316:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1414 00000100:00080000:0.0:1775765526.151322:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765526.151327:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765526.151339:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765526.151356:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741869440:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765526.151358:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741869568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765526.151378:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765526.151381:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765526.151383:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765526.151387:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1414 00000100:00080000:0.0:1775765526.151390:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765526.151393:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765526.151397:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765526.151407:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741869696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765526.151416:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765526.151420:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765526.151422:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765526.151425:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1414 00000100:00080000:0.0:1775765526.151428:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765526.151431:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765526.151435:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765526.151451:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741869824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765526.151456:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765526.151512:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765526/real 1775765526] req@ffff90c20363a680 x1862023741869440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765526.151532:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741869440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765526.151544:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765526.151550:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765526.151555:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741869440:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765526.151590:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765526.151618:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765526.151630:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765526/real 1775765526] req@ffff90c20363aa00 x1862023741869568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765526.151642:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741869568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765526.151653:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765526.151658:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765526.151663:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765526.151672:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765526.151698:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765526.151712:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765526/real 1775765526] req@ffff90c20363b100 x1862023741869696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765526.151729:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741869696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765526.151740:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765526.151745:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765526.151750:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765526.151759:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869824:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765526.151782:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765526.151792:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765526/real 1775765526] req@ffff90c20363bb80 x1862023741869824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765526.151802:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741869824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765526.151813:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765526.151815:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765526.151818:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741869824:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765527.175151:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765528.199180:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765529.223024:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765530.247121:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765531.271004:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765531.271013:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765531.271017:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765531.271026:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1419 00010000:00010000:1.0:1775765531.271027:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765531.271049:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765531.271056:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765531.271091:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741869952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765531.271123:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765531.271126:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765531.271127:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765531.271130:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1419 00000100:00080000:0.0:1775765531.271132:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765531.271135:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765531.271142:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765531.271150:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741870080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765531.271158:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741869952:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765531.271160:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765531.271165:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765531.271167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765531.271170:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1419 00000100:00080000:0.0:1775765531.271172:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765531.271174:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765531.271177:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765531.271195:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741870208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765531.271202:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765531.271205:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765531.271206:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765531.271209:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1419 00000100:00080000:0.0:1775765531.271211:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765531.271213:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765531.271216:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765531.271224:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741870336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765531.271232:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765531.271266:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765531/real 1775765531] req@ffff90c203638380 x1862023741869952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765531.271281:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741869952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765531.271289:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765531.271292:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765531.271295:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741869952:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765531.271311:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870080:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765531.271325:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765531.271333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765531/real 1775765531] req@ffff90c203638e00 x1862023741870080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765531.271340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741870080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765531.271346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765531.271348:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765531.271350:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765531.271354:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870208:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765531.271363:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765531.271368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765531/real 1775765531] req@ffff90c20363b480 x1862023741870208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765531.271375:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741870208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765531.271384:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765531.271386:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765531.271388:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765531.271391:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870336:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765531.271401:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765531.271405:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765531/real 1775765531] req@ffff90c20363b800 x1862023741870336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765531.271412:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741870336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765531.271418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765531.271420:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765531.271422:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870336:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765532.295162:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765533.319079:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765534.343114:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765535.367118:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765536.391126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765536.391170:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765536.391179:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765536.391206:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1424 00010000:00010000:1.0:1775765536.391212:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765536.391233:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765536.391244:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765536.391297:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741870464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765536.391336:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765536.391341:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765536.391344:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765536.391349:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1424 00000100:00080000:3.0:1775765536.391353:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765536.391358:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765536.391372:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765536.391388:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741870592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765536.391400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741870464:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765536.391408:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765536.391415:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765536.391417:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765536.391423:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1424 00000100:00080000:3.0:1775765536.391426:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765536.391431:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765536.391439:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765536.391456:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741870720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765536.391469:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765536.391473:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765536.391475:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765536.391492:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1424 00000100:00080000:3.0:1775765536.391496:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765536.391500:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765536.391507:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765536.391510:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765536.391530:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741870848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765536.391573:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765536/real 1775765536] req@ffff90c218da0a80 x1862023741870464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765536.391599:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741870464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765536.391623:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765536.391629:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765536.391634:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741870464:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765536.391664:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870592:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765536.391689:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765536.391704:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765536/real 1775765536] req@ffff90c218da0000 x1862023741870592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765536.391718:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741870592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765536.391728:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765536.391733:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765536.391737:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765536.391743:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765536.391760:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765536.391770:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765536/real 1775765536] req@ffff90c218da1f80 x1862023741870720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765536.391785:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741870720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765536.391800:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765536.391806:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765536.391813:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765536.391824:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870848:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765536.391861:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765536.391926:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765536/real 1775765536] req@ffff90c218da3100 x1862023741870848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765536.391946:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741870848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765536.391959:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765536.391962:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765536.391966:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741870848:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765537.415069:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765538.438985:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765539.463178:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765540.487155:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765541.511178:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765541.511193:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765541.511200:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765541.511215:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1429 00000100:00080000:3.0:1775765541.511233:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765541.511245:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765541.511276:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765541.511307:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741870976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765541.511345:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765541.511350:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765541.511353:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765541.511359:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1429 00000100:00080000:3.0:1775765541.511363:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765541.511367:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765541.511382:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765541.511398:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741871104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765541.511413:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765541.511419:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741870976:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765541.511419:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765541.511423:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765541.511436:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1429 00000100:00080000:3.0:1775765541.511440:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765541.511444:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765541.511449:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765541.511466:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741871232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765541.511477:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765541.511482:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765541.511484:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765541.511489:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1429 00000100:00080000:3.0:1775765541.511532:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765541.511538:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765541.511547:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765541.511555:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765541.511578:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741871360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765541.511637:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765541/real 1775765541] req@ffff90c218da0700 x1862023741870976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765541.511669:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741870976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765541.511684:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765541.511690:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765541.511696:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741870976:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765541.511730:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765541.511760:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765541.511774:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765541/real 1775765541] req@ffff90c218da3480 x1862023741871104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765541.511806:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741871104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765541.511823:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765541.511827:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765541.511830:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765541.511836:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765541.511925:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765541.511941:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765541/real 1775765541] req@ffff90c218da0380 x1862023741871232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765541.511955:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741871232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765541.511964:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765541.511968:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765541.511972:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765541.511978:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871360:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765541.512002:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765541.512012:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765541/real 1775765541] req@ffff90c218da2d80 x1862023741871360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765541.512025:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741871360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765541.512036:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765541.512039:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765541.512044:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871360:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765542.535176:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765543.559046:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765544.583164:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765545.607130:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765546.631017:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765546.631063:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765546.631073:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765546.631087:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1435 00000100:00080000:3.0:1775765546.631102:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765546.631111:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765546.631182:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741871488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765546.631212:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765546.631218:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765546.631222:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765546.631232:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1435 00000100:00080000:3.0:1775765546.631238:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765546.631244:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765546.631255:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00100000:3.0:1775765546.631258:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765546.631282:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741871616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765546.631289:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741871488:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765546.631331:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765546.631336:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765546.631338:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765546.631344:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1435 00000100:00080000:3.0:1775765546.631348:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765546.631352:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765546.631363:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765546.631376:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741871744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765546.631390:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765546.631395:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765546.631399:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765546.631403:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1435 00000100:00080000:3.0:1775765546.631422:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765546.631426:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765546.631433:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765546.631441:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765546.631456:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741871872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765546.631513:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765546/real 1775765546] req@ffff90c218da3800 x1862023741871488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765546.631540:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741871488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765546.631554:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765546.631560:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765546.631565:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741871488:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765546.631599:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871616:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765546.631624:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765546.631637:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765546/real 1775765546] req@ffff90c218da1880 x1862023741871616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765546.631650:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741871616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765546.631660:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765546.631664:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765546.631668:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765546.631674:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871744:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765546.631691:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765546.631701:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765546/real 1775765546] req@ffff90c218da0e00 x1862023741871744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765546.631713:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741871744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765546.631721:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765546.631724:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765546.631727:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765546.631733:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871872:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765546.631750:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765546.631759:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765546/real 1775765546] req@ffff90c218da2d80 x1862023741871872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765571 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765546.631769:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741871872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765571 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765546.631781:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765546.631784:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765546.631788:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741871872:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765547.655204:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765548.679191:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765549.703139:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765550.727222:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765551.751047:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765551.751057:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765551.751063:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765551.751073:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1440 00000100:00080000:3.0:1775765551.751086:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765551.751094:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765551.751138:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741872000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765551.751166:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765551.751169:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765551.751174:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765551.751177:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765551.751183:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1440 00000100:00080000:3.0:1775765551.751187:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765551.751201:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765551.751212:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765551.751229:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741872128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765551.751230:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741872000:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765551.751248:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765551.751251:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765551.751253:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765551.751256:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1440 00000100:00080000:3.0:1775765551.751259:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765551.751262:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765551.751267:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765551.751280:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741872256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765551.751288:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765551.751291:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765551.751292:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765551.751297:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1440 00000100:00080000:3.0:1775765551.751300:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765551.751303:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765551.751309:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765551.751309:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765551.751327:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741872384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765551.751369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765551/real 1775765551] req@ffff90c218da0380 x1862023741872000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765551.751388:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741872000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765551.751399:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765551.751403:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765551.751407:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741872000:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765551.751429:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765551.751447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765551.751457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765551/real 1775765551] req@ffff90c218da3480 x1862023741872128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765551.751485:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741872128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765551.751493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765551.751496:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765551.751499:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765551.751504:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765551.751519:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765551.751527:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765551/real 1775765551] req@ffff90c218da0700 x1862023741872256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765551.751537:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741872256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765551.751544:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765551.751546:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765551.751549:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765551.751554:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872384:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765551.751568:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765551.751576:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765551/real 1775765551] req@ffff90c218da3100 x1862023741872384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765576 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765551.751586:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741872384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765576 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765551.751595:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765551.751597:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765551.751600:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872384:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765552.775211:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765553.799192:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765554.823220:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765555.847192:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765556.871099:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765556.871115:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:1.0:1775765556.871123:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765556.871124:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765556.871154:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1445 00000100:00080000:3.0:1775765556.871175:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765556.871185:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765556.871265:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741872512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765556.871323:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765556.871330:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765556.871334:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765556.871344:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1445 00000100:00080000:3.0:1775765556.871353:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765556.871362:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765556.871381:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765556.871397:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741872512:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765556.871416:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741872640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765556.871444:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765556.871449:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765556.871452:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765556.871461:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1445 00000100:00080000:3.0:1775765556.871468:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765556.871475:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765556.871485:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765556.871502:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741872768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765556.871515:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765556.871519:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765556.871522:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765556.871540:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1445 00000800:00000100:2.0:1775765556.871546:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765556.871560:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765556.871566:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765556.871576:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765556.871596:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741872896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765556.871644:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765556/real 1775765556] req@ffff90c218da1f80 x1862023741872512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765556.871679:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741872512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765556.871693:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765556.871698:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765556.871703:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741872512:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765556.871738:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765556.871770:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765556.871785:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765556/real 1775765556] req@ffff90c218da0000 x1862023741872640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765556.871801:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741872640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765556.871811:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765556.871815:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765556.871819:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765556.871826:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765556.871845:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765556.871854:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765556/real 1775765556] req@ffff90c218da0a80 x1862023741872768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765556.871867:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741872768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765556.871876:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765556.871879:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765556.871883:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765556.871935:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872896:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765556.871996:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765556.872017:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765556/real 1775765556] req@ffff90c218da3100 x1862023741872896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765556.872038:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741872896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765581 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765556.872057:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765556.872061:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765556.872065:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741872896:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765557.895166:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765558.919092:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765559.943127:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765560.967122:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765561.991059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765561.991070:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765561.991080:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765561.991086:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765561.991100:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1450 00000100:00080000:0.0:1775765561.991116:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765561.991124:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765561.991175:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741873024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765561.991204:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765561.991207:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765561.991211:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765561.991216:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1450 00000100:00080000:0.0:1775765561.991221:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765561.991226:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765561.991238:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741873024:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775765561.991239:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765561.991262:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741873152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765561.991275:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765561.991278:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765561.991280:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765561.991284:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1450 00000100:00080000:0.0:1775765561.991288:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765561.991293:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765561.991299:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765561.991313:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741873280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765561.991324:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765561.991329:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765561.991331:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765561.991344:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1450 00000100:00080000:0.0:1775765561.991349:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765561.991353:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765561.991357:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765561.991369:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741873408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765561.991392:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765561.991426:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765561.991443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873408:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765563.015131:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765564.039080:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765565.063124:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765565.063150:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775765565.063191:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765565.063209:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765565.063245:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765565.063259:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765565.063280:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775765565.063300:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765561/real 1775765565] req@ffff90c20363b800 x1862023741873152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765565.063349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741873152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765565.063362:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765565.063367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765565.063372:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765565.063379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765561/real 1775765565] req@ffff90c20363b480 x1862023741873280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765565.063391:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741873280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765565.063396:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765565.063401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765565.063404:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765565.063410:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765561/real 1775765565] req@ffff90c203638e00 x1862023741873408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765565.063419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741873408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765565.063428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765565.063431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765565.063433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873408:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775765565.063439:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765561/real 1775765565] req@ffff90c203639f80 x1862023741873024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765565.063448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741873024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765586 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765565.063454:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765565.063457:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765565.063461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741873024:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775765566.087064:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765567.111073:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765567.111087:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765567.111103:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765567.111135:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1455 00000100:00080000:3.0:1775765567.111150:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765567.111160:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765567.111209:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741873536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765567.111237:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765567.111242:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765567.111246:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765567.111254:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1455 00000100:00080000:3.0:1775765567.111259:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765567.111265:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765567.111276:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00100000:3.0:1775765567.111277:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765567.111298:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741873664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765567.111304:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741873536:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765567.111329:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765567.111336:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765567.111339:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765567.111349:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1455 00000100:00080000:3.0:1775765567.111356:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765567.111364:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765567.111392:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765567.111428:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741873792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765567.111429:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765567.111458:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765567.111463:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765567.111465:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765567.111489:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1455 00000100:00080000:3.0:1775765567.111495:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765567.111501:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765567.111508:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765567.111516:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765567/real 1775765567] req@ffff90c218da0700 x1862023741873536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765567.111529:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741873920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765567.111583:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741873536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765567.111604:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765567.111611:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765567.111619:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741873536:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765567.111657:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765567.111698:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765567.111734:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765567/real 1775765567] req@ffff90c218da3480 x1862023741873664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765567.111757:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741873664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765567.111773:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765567.111779:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765567.111784:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765567.111796:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765567.111832:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765567.111934:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765567/real 1775765567] req@ffff90c218da0380 x1862023741873792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765567.111958:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741873792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765567.111972:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765567.111978:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765567.111984:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765567.111996:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873920:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765567.112033:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765567.112051:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765567/real 1775765567] req@ffff90c218da2d80 x1862023741873920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765567.112070:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741873920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765567.112088:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765567.112094:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765567.112099:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741873920:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765568.135155:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765569.159087:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765570.182992:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765571.207067:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765572.230930:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765572.230941:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765572.230948:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765572.230960:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1460 00000100:00080000:3.0:1775765572.230975:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765572.230986:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765572.231000:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765572.231033:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741874048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765572.231070:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765572.231074:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765572.231075:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765572.231079:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1460 00000100:00080000:3.0:1775765572.231082:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765572.231086:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765572.231097:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765572.231106:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741874048:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765572.231110:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741874176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765572.231121:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765572.231123:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765572.231125:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765572.231128:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1460 00000100:00080000:3.0:1775765572.231130:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765572.231133:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765572.231136:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765572.231145:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741874304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765572.231151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765572.231154:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765572.231155:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765572.231159:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1460 00000100:00080000:3.0:1775765572.231178:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765572.231180:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765572.231183:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765572.231191:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765572.231194:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741874432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765572.231234:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765572/real 1775765572] req@ffff90c218da0e00 x1862023741874048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765572.231261:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741874048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765572.231271:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765572.231275:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765572.231279:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741874048:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765572.231304:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874176:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765572.231322:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765572.231343:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765572/real 1775765572] req@ffff90c218da1880 x1862023741874176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765572.231352:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741874176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765572.231357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765572.231359:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765572.231361:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765572.231365:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874304:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765572.231376:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765572.231381:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765572/real 1775765572] req@ffff90c218da3800 x1862023741874304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765572.231389:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741874304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765572.231394:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765572.231396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765572.231398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765572.231401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874432:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765572.231411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765572.231416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765572/real 1775765572] req@ffff90c218da2d80 x1862023741874432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765572.231422:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741874432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765572.231434:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765572.231437:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765572.231441:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874432:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765573.255070:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765574.279080:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765575.303001:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765576.326991:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765577.350974:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765577.350985:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765577.350992:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765577.351002:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1465 00000100:00080000:3.0:1775765577.351014:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765577.351022:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765577.351028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765577.351062:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741874560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765577.351089:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765577.351093:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765577.351094:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765577.351099:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1465 00000100:00080000:3.0:1775765577.351103:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765577.351107:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765577.351117:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765577.351124:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741874560:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765577.351131:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741874688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765577.351143:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765577.351147:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765577.351150:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765577.351154:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1465 00000100:00080000:3.0:1775765577.351172:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765577.351176:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765577.351181:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765577.351181:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765577.351195:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741874816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765577.351213:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765577.351217:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775765577.351219:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765577/real 1775765577] req@ffff90c218da0380 x1862023741874560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765577.351220:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765577.351229:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1465 00000100:00080000:3.0:1775765577.351236:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765577.351240:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765577.351244:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741874560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775765577.351246:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775765577.351257:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765577.351261:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:3.0:1775765577.351265:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741874944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765577.351266:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741874560:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765577.351290:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765577.351306:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765577.351314:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765577/real 1775765577] req@ffff90c218da3480 x1862023741874688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765577.351322:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741874688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765577.351328:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765577.351330:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765577.351332:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765577.351336:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765577.351346:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765577.351351:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765577/real 1775765577] req@ffff90c218da0700 x1862023741874816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765577.351359:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741874816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765577.351364:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765577.351365:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765577.351367:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765577.351371:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874944:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765577.351380:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765577.351385:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765577/real 1775765577] req@ffff90c218da3100 x1862023741874944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765577.351391:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741874944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765577.351397:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765577.351399:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765577.351401:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741874944:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765578.375078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765579.399020:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765580.423085:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765581.447008:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765582.470989:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765582.470999:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765582.471005:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765582.471015:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1470 00000100:00080000:3.0:1775765582.471027:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765582.471034:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765582.471091:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765582.471092:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741875072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765582.471121:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765582.471125:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765582.471127:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765582.471131:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1470 00000100:00080000:3.0:1775765582.471136:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765582.471146:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765582.471156:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765582.471165:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741875072:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765582.471173:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741875200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765582.471184:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765582.471188:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765582.471190:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765582.471194:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1470 00000100:00080000:3.0:1775765582.471197:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765582.471200:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765582.471204:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765582.471214:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741875328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765582.471223:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765582.471226:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765582.471228:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765582.471232:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1470 00000800:00000100:2.0:1775765582.471238:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765582.471239:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765582.471246:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765582.471251:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765582.471267:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741875456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765582.471283:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765582/real 1775765582] req@ffff90c218da0a80 x1862023741875072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765582.471317:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741875072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765582.471338:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765582.471344:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765582.471349:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741875072:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765582.471378:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765582.471409:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765582.471428:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765582/real 1775765582] req@ffff90c218da0000 x1862023741875200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765582.471445:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741875200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765582.471458:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765582.471463:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765582.471469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765582.471475:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765582.471496:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765582.471507:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765582/real 1775765582] req@ffff90c218da1f80 x1862023741875328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765582.471517:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741875328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765582.471531:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765582.471534:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765582.471536:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765582.471542:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875456:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765582.471555:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765582.471562:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765582/real 1775765582] req@ffff90c218da3100 x1862023741875456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765582.471572:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741875456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765582.471580:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765582.471583:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765582.471585:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875456:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765583.495005:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765584.519060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765585.543058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765586.567133:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765587.591001:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765587.591013:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765587.591018:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765587.591029:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1476 00000100:00080000:3.0:1775765587.591042:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765587.591050:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765587.591096:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741875584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765587.591135:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765587.591139:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765587.591142:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765587.591148:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1476 00000100:00080000:3.0:1775765587.591153:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765587.591159:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765587.591175:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00100000:3.0:1775765587.591180:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765587.591197:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741875584:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765587.591198:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741875712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765587.591222:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765587.591227:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765587.591231:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765587.591238:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1476 00000100:00080000:3.0:1775765587.591244:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765587.591251:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765587.591258:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765587.591282:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741875840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765587.591283:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765587.591304:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765587.591310:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765587.591313:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765587.591320:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1476 00000100:00100000:2.0:1775765587.591332:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765587/real 1775765587] req@ffff90c218da0700 x1862023741875584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765587.591340:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765587.591352:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765587.591362:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765587.591377:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741875584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765587.591383:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741875968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765587.591434:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765587.591442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765587.591448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741875584:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765587.591488:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875712:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765587.591517:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765587.591532:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765587/real 1775765587] req@ffff90c218da3480 x1862023741875712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765587.591562:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741875712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765587.591570:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765587.591573:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765587.591577:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765587.591582:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875840:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765587.591598:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765587.591606:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765587/real 1775765587] req@ffff90c218da0380 x1862023741875840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765587.591616:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741875840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765587.591623:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765587.591626:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765587.591628:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765587.591689:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875968:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765587.591720:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765587.591742:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765587/real 1775765587] req@ffff90c218da2d80 x1862023741875968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765587.591760:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741875968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765587.591776:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765587.591781:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765587.591786:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741875968:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765588.615068:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765589.639038:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765590.663144:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765591.687005:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765592.710996:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765592.711005:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765592.711010:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765592.711020:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1481 00000100:00080000:3.0:1775765592.711032:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765592.711040:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765592.711077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765592.711086:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741876096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765592.711116:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765592.711120:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765592.711122:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765592.711125:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1481 00000100:00080000:3.0:1775765592.711128:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765592.711147:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765592.711158:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765592.711172:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741876224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765592.711183:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765592.711187:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765592.711189:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765592.711193:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1481 00000100:00080000:3.0:1775765592.711197:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765592.711200:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765592.711206:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765592.711213:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741876096:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765592.711224:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741876352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765592.711241:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765592.711247:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765592.711251:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765592.711257:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1481 00000100:00080000:3.0:1775765592.711310:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765592.711316:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765592.711322:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765592.711328:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765592.711345:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741876480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765592.711372:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765592/real 1775765592] req@ffff90c218da3800 x1862023741876096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765592.711390:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741876096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765592.711400:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765592.711404:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765592.711408:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741876096:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765592.711414:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765592.711432:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765592.711441:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765592/real 1775765592] req@ffff90c218da1880 x1862023741876224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765592.711451:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741876224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765592.711462:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765592.711465:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765592.711468:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765592.711488:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765592.711500:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765592.711507:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765592/real 1775765592] req@ffff90c218da0e00 x1862023741876352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765592.711516:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741876352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765592.711522:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765592.711524:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765592.711526:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765592.711530:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876480:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765592.711543:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765592.711549:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765592/real 1775765592] req@ffff90c218da2d80 x1862023741876480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765617 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765592.711558:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741876480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765617 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765592.711565:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765592.711567:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765592.711570:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876480:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765593.735143:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765594.759100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765595.783014:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765596.807079:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765597.831011:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765597.831024:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765597.831029:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765597.831041:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1486 00000100:00080000:3.0:1775765597.831056:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765597.831065:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765597.831113:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741876608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765597.831126:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765597.831157:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765597.831173:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765597.831175:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765597.831181:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1486 00000100:00080000:3.0:1775765597.831185:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765597.831191:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765597.831202:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765597.831218:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741876736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765597.831233:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765597.831237:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741876608:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765597.831239:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765597.831242:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765597.831261:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1486 00000100:00080000:3.0:1775765597.831265:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765597.831269:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765597.831275:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765597.831294:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741876864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765597.831305:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765597.831309:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765597.831311:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765597.831317:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1486 00000100:00080000:3.0:1775765597.831346:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765597.831350:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765597.831356:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765597.831358:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765597.831381:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741876992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765597.831424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765597/real 1775765597] req@ffff90c218da0380 x1862023741876608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765597.831455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741876608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765597.831467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765597.831472:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765597.831476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741876608:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765597.831507:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876736:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765597.831527:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765597.831537:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765597/real 1775765597] req@ffff90c218da3480 x1862023741876736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765597.831569:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741876736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765597.831582:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765597.831586:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765597.831591:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765597.831611:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876864:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765597.831639:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765597.831654:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765597/real 1775765597] req@ffff90c218da0700 x1862023741876864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765597.831664:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741876864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765597.831671:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765597.831674:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765597.831677:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765597.831683:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876992:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765597.831700:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765597.831707:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765597/real 1775765597] req@ffff90c218da3100 x1862023741876992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765597.831716:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741876992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765622 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765597.831725:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765597.831727:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765597.831730:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741876992:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765598.855042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765599.879084:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765600.903044:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765601.927093:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765602.950999:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765602.951011:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765602.951018:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765602.951032:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1491 00000100:00080000:0.0:1775765602.951048:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765602.951058:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765602.951063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765602.951115:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741877120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765602.951170:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765602.951175:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765602.951177:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765602.951182:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1491 00000100:00080000:0.0:1775765602.951187:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765602.951192:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765602.951206:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765602.951227:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741877248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765602.951228:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741877120:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765602.951252:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765602.951256:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765602.951258:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765602.951264:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1491 00000100:00080000:0.0:1775765602.951268:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765602.951272:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765602.951278:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765602.951294:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741877376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765602.951305:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765602.951309:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765602.951311:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765602.951315:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1491 00000100:00080000:0.0:1775765602.951320:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765602.951324:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765602.951329:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765602.951338:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765602.951344:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741877504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765602.951416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765602/real 1775765602] req@ffff90c203638380 x1862023741877120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765602.951439:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741877120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765602.951451:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765602.951456:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765602.951461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741877120:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765602.951487:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765602.951539:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765602.951552:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765602/real 1775765602] req@ffff90c20363bb80 x1862023741877248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765602.951563:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741877248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765602.951571:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765602.951575:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765602.951578:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765602.951583:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765602.951598:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765602.951606:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765602/real 1775765602] req@ffff90c20363b100 x1862023741877376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765602.951616:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741877376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765602.951623:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765602.951625:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765602.951628:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765602.951633:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877504:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765602.951648:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765602.951655:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765602/real 1775765602] req@ffff90c20363aa00 x1862023741877504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765602.951665:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741877504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765627 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765602.951675:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765602.951677:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765602.951679:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877504:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765603.975073:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765604.999042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765606.023210:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765607.047204:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765608.071113:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765608.071127:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765608.071136:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765608.071165:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1496 00000100:00080000:0.0:1775765608.071179:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765608.071189:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765608.071237:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741877632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765608.071243:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765608.071287:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765608.071291:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765608.071294:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765608.071298:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1496 00000100:00080000:0.0:1775765608.071302:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765608.071305:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765608.071318:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765608.071333:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741877760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765608.071347:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765608.071353:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765608.071355:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765608.071379:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1496 00000100:00100000:2.0:1775765608.071383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741877632:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765608.071385:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765608.071395:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765608.071411:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765608.071421:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741877888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765608.071432:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765608.071435:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765608.071438:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765608.071444:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1496 00000100:00080000:0.0:1775765608.071448:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765608.071477:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765608.071484:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765608.071499:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741878016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765608.071563:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765608.071623:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765608/real 1775765608] req@ffff90c20363a680 x1862023741877632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765608.071648:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741877632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765608.071662:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765608.071667:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765608.071671:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741877632:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765608.071702:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765608.071737:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765608.071749:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765608/real 1775765608] req@ffff90c203638000 x1862023741877760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765608.071762:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741877760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765608.071770:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765608.071773:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765608.071776:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765608.071782:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765608.071797:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765608.071813:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765608/real 1775765608] req@ffff90c203639500 x1862023741877888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765608.071840:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741877888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765608.071850:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765608.071854:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765608.071858:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741877888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765608.071865:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878016:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765608.071887:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765608.071897:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765608/real 1775765608] req@ffff90c203639180 x1862023741878016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765608.071960:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741878016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765608.071983:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765608.071987:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765608.071991:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878016:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765609.095131:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765610.119130:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765611.143153:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765612.167073:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765613.191060:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765613.191070:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765613.191076:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765613.191087:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1501 00000100:00080000:3.0:1775765613.191122:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765613.191130:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765613.191160:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765613.191170:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741878144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765613.191200:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765613.191203:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765613.191205:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765613.191208:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1501 00000100:00080000:3.0:1775765613.191212:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765613.191216:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765613.191225:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765613.191235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741878144:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765613.191237:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741878272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765613.191250:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765613.191253:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765613.191254:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765613.191257:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1501 00000100:00080000:3.0:1775765613.191260:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765613.191262:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765613.191266:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765613.191275:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741878400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765613.191282:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765613.191284:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765613.191285:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765613.191288:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1501 00000100:00080000:3.0:1775765613.191292:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765613.191295:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765613.191299:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765613.191302:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765613.191319:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741878528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765613.191346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765613/real 1775765613] req@ffff90c218da1f80 x1862023741878144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765613.191365:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741878144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765613.191374:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765613.191379:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765613.191383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741878144:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765613.191427:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878272:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765613.191465:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765613.191476:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765613/real 1775765613] req@ffff90c218da0000 x1862023741878272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765613.191486:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741878272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765613.191493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765613.191496:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765613.191498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765613.191504:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878400:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765613.191516:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765613.191523:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765613/real 1775765613] req@ffff90c218da0a80 x1862023741878400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765613.191532:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741878400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765613.191538:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765613.191540:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765613.191542:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765613.191547:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878528:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765613.191558:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765613.191564:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765613/real 1775765613] req@ffff90c218da3100 x1862023741878528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765613.191572:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741878528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765613.191580:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765613.191582:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765613.191584:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878528:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765614.215118:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765615.239077:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765616.263098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765617.287128:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765618.311014:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765618.311027:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765618.311042:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775765618.311050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765618.311054:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1506 00000100:00080000:0.0:1775765618.311066:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765618.311075:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765618.311119:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741878656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765618.311145:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765618.311149:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765618.311150:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765618.311154:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1506 00000100:00080000:0.0:1775765618.311159:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765618.311164:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765618.311174:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765618.311192:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741878784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765618.311194:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741878656:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765618.311228:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765618.311233:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765618.311235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765618.311240:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1506 00000100:00080000:0.0:1775765618.311243:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765618.311247:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765618.311250:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765618.311262:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741878912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765618.311270:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765618.311272:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765618.311275:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765618.311279:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1506 00000100:00080000:0.0:1775765618.311282:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765618.311285:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765618.311289:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765618.311297:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765618.311307:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741879040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765618.311346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765618/real 1775765618] req@ffff90c203638700 x1862023741878656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765618.311382:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741878656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765618.311392:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765618.311395:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765618.311399:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741878656:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765618.311424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765618.311444:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765618.311453:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765618/real 1775765618] req@ffff90c203639180 x1862023741878784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765618.311461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741878784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765618.311467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765618.311469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765618.311471:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765618.311475:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765618.311486:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765618.311491:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765618/real 1775765618] req@ffff90c203639500 x1862023741878912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765618.311498:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741878912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765618.311508:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765618.311510:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765618.311513:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741878912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765618.311516:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879040:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765618.311527:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765618.311532:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765618/real 1775765618] req@ffff90c203638000 x1862023741879040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765618.311539:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741879040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765618.311545:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765618.311547:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765618.311549:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879040:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765619.335050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765620.359034:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765621.383155:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765622.407069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765623.431039:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765623.431053:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765623.431059:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765623.431072:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1511 00010000:00010000:1.0:1775765623.431073:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765623.431086:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765623.431096:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765623.431141:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741879168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765623.431181:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765623.431188:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765623.431191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765623.431198:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1511 00000100:00080000:3.0:1775765623.431206:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765623.431213:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765623.431229:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765623.431240:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741879168:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765623.431260:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741879296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765623.431283:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765623.431290:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765623.431293:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765623.431300:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1511 00000100:00080000:3.0:1775765623.431393:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765623.431401:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765623.431409:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765623.431417:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765623.431439:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741879424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765623.431452:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765623.431456:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765623.431459:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765623.431464:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765623/real 1775765623] req@ffff90c218da0700 x1862023741879168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765623.431468:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1511 00000100:00080000:3.0:1775765623.431478:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765623.431488:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765623.431496:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765623.431506:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741879168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765623.431514:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741879552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765623.431528:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765623.431539:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765623.431547:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741879168:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765623.431577:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765623.431605:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765623.431618:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765623/real 1775765623] req@ffff90c218da3480 x1862023741879296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765623.431632:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741879296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765623.431642:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765623.431646:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765623.431650:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765623.431656:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879424:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765623.431672:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765623.431681:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765623/real 1775765623] req@ffff90c218da0380 x1862023741879424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765623.431693:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741879424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765623.431701:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765623.431704:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765623.431708:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765623.431713:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879552:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765623.431729:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765623.431737:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765623/real 1775765623] req@ffff90c218da2d80 x1862023741879552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765623.431750:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741879552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765623.431760:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765623.431763:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765623.431766:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879552:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765624.455108:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765625.479131:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765626.503115:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765627.527088:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765628.550999:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765628.551061:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765628.551068:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765628.551073:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765628.551084:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1517 00000100:00080000:0.0:1775765628.551097:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765628.551103:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765628.551149:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741879680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765628.551178:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765628.551181:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765628.551182:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765628.551185:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1517 00000100:00080000:0.0:1775765628.551188:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765628.551191:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765628.551201:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765628.551212:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741879808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765628.551225:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741879680:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765628.551226:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765628.551239:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765628.551246:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765628.551249:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1517 00000100:00080000:0.0:1775765628.551252:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765628.551270:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765628.551275:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765628.551291:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741879936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765628.551298:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765628.551301:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765628.551302:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765628.551306:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1517 00000100:00080000:0.0:1775765628.551310:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765628.551314:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765628.551317:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765628.551325:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741880064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765628.551364:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765628.551400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765628.551414:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880064:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765629.575205:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765630.599074:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765631.623179:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775765631.623336:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775765631.623406:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765631.623435:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775765631.623505:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775765631.623528:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775765631.623548:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775765631.623569:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765628/real 1775765631] req@ffff90c20363aa00 x1862023741879808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765631.623621:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741879808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765631.623644:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765631.623652:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765631.623660:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765631.623676:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765628/real 1775765631] req@ffff90c20363b100 x1862023741879936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765631.623721:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741879936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765631.623738:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765631.623745:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765631.623752:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741879936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765631.623765:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765628/real 1775765631] req@ffff90c20363bb80 x1862023741880064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765631.623789:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741880064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765631.623821:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765631.623828:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765631.623834:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880064:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775765631.623846:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765628/real 1775765631] req@ffff90c20363a680 x1862023741879680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765631.623907:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741879680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765631.623925:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765631.623933:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765631.623941:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741879680:192.168.202.155@tcp:250:kworker.0 00010000:00010000:0.0:1775765632.647115:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765633.671047:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765633.671057:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765633.671061:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765633.671071:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1522 00000100:00080000:3.0:1775765633.671083:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765633.671089:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765633.671127:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741880192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765633.671157:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765633.671161:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765633.671163:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765633.671166:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1522 00000100:00080000:3.0:1775765633.671169:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765633.671172:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765633.671184:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00010000:00010000:1.0:1775765633.671188:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765633.671224:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741880320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765633.671239:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741880192:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765633.671248:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765633.671254:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765633.671256:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765633.671263:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1522 00000100:00080000:3.0:1775765633.671270:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765633.671275:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765633.671284:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765633.671303:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741880448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765633.671317:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765633.671324:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1775765633.671325:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765633.671328:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765633.671336:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1522 00000100:00080000:3.0:1775765633.671343:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765633.671349:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765633.671356:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765633.671379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765633/real 1775765633] req@ffff90c218da0e00 x1862023741880192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765633.671397:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741880576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765633.671410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741880192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765633.671428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765633.671434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765633.671442:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741880192:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765633.671492:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765633.671524:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765633.671542:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765633/real 1775765633] req@ffff90c218da1880 x1862023741880320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765633.671558:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741880320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765633.671569:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765633.671573:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765633.671578:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765633.671586:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765633.671610:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765633.671623:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765633/real 1775765633] req@ffff90c218da3800 x1862023741880448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765633.671638:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741880448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765633.671656:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765633.671661:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765633.671666:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765633.671674:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880576:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765633.671699:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765633.671711:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765633/real 1775765633] req@ffff90c218da2d80 x1862023741880576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765658 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765633.671727:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741880576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765658 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765633.671740:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765633.671745:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765633.671749:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880576:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765634.695165:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765635.719077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765636.743108:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765637.767252:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765638.790995:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765638.791005:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765638.791011:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765638.791021:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1527 00000100:00080000:0.0:1775765638.791033:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765638.791042:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765638.791104:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765638.791119:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741880704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765638.791168:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765638.791172:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765638.791174:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765638.791179:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1527 00000100:00080000:0.0:1775765638.791183:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765638.791187:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765638.791197:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765638.791212:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741880832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765638.791218:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741880704:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765638.791230:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765638.791234:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765638.791235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765638.791239:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1527 00000100:00080000:0.0:1775765638.791242:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765638.791245:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765638.791249:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765638.791259:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741880960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765638.791268:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765638.791271:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765638.791272:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765638.791275:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1527 00000100:00080000:0.0:1775765638.791285:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765638.791288:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765638.791292:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765638.791294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765638.791305:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741881088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765638.791341:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765638/real 1775765638] req@ffff90c203638380 x1862023741880704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765638.791363:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741880704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765638.791374:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765638.791378:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765638.791383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741880704:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765638.791418:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880832:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765638.791441:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765638.791454:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765638/real 1775765638] req@ffff90c203639f80 x1862023741880832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765638.791488:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741880832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765638.791497:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765638.791501:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765638.791505:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765638.791512:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880960:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765638.791533:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765638.791545:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765638/real 1775765638] req@ffff90c203638e00 x1862023741880960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765638.791558:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741880960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765638.791567:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765638.791571:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765638.791574:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741880960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765638.791581:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881088:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765638.791609:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765638.791620:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765638/real 1775765638] req@ffff90c20363b480 x1862023741881088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765663 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765638.791635:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741881088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765663 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765638.791647:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765638.791651:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765638.791654:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881088:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765639.815096:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765640.839061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765641.863014:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765642.887097:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765643.910966:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765643.910973:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765643.910978:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765643.910987:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1532 00000100:00080000:3.0:1775765643.910996:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765643.911002:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765643.911038:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741881216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765643.911058:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:1.0:1775765643.911067:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765643.911068:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765643.911071:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765643.911075:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1532 00000100:00080000:3.0:1775765643.911078:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765643.911082:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765643.911090:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765643.911094:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741881216:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765643.911102:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741881344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765643.911133:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765643.911136:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765643.911138:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765643.911141:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1532 00000100:00080000:3.0:1775765643.911144:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765643.911147:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765643.911151:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765643.911160:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741881472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765643.911166:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765643.911169:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765643.911171:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765643.911173:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1532 00000100:00080000:3.0:1775765643.911175:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765643.911177:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765643.911182:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765643.911183:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765643.911193:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741881600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765643.911224:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765643/real 1775765643] req@ffff90c218da0380 x1862023741881216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765643.911238:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741881216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765643.911246:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765643.911249:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765643.911253:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741881216:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765643.911278:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765643.911294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765643.911301:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765643/real 1775765643] req@ffff90c218da3480 x1862023741881344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765643.911309:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741881344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765643.911319:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765643.911321:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765643.911323:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765643.911327:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765643.911336:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765643.911341:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765643/real 1775765643] req@ffff90c218da0700 x1862023741881472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765643.911348:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741881472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765643.911353:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765643.911355:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765643.911357:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765643.911360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881600:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765643.911369:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765643.911374:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765643/real 1775765643] req@ffff90c218da3100 x1862023741881600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765643.911380:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741881600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765668 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765643.911386:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765643.911388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765643.911390:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881600:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765644.935058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765645.958999:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765646.983152:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765648.007089:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765649.031037:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765649.031047:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765649.031053:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765649.031089:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1537 00000100:00080000:3.0:1775765649.031101:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765649.031108:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765649.031159:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741881728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765649.031195:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765649.031199:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765649.031201:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765649.031206:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1537 00000100:00080000:3.0:1775765649.031210:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765649.031214:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765649.031233:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00010000:00010000:1.0:1775765649.031247:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765649.031249:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741881856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765649.031265:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765649.031268:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765649.031270:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765649.031278:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1537 00000100:00080000:3.0:1775765649.031289:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775765649.031290:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741881728:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765649.031315:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765649.031322:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765649.031335:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741881984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765649.031346:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765649.031349:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765649.031351:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765649.031355:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1537 00000100:00080000:3.0:1775765649.031359:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765649.031363:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765649.031368:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765649.031378:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741882112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765649.031423:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765649.031482:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765649/real 1775765649] req@ffff90c218da0a80 x1862023741881728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765649.031517:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741881728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765649.031531:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765649.031537:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765649.031542:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741881728:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765649.031586:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765649.031610:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765649.031624:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765649/real 1775765649] req@ffff90c218da0000 x1862023741881856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765649.031638:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741881856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765649.031648:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765649.031652:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765649.031655:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765649.031662:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765649.031679:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765649.031688:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765649/real 1775765649] req@ffff90c218da1f80 x1862023741881984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765649.031700:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741881984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765649.031726:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765649.031730:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765649.031733:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741881984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765649.031739:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882112:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765649.031755:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765649.031768:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765649/real 1775765649] req@ffff90c218da3100 x1862023741882112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765649.031791:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741882112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765649.031808:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765649.031815:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765649.031821:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882112:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765650.055185:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765651.079168:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765652.103103:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765653.127069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765654.151079:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765654.151087:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765654.151092:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765654.151101:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1542 00000100:00080000:0.0:1775765654.151112:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765654.151119:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765654.151159:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741882240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765654.151159:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765654.151190:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765654.151193:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765654.151195:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765654.151198:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1542 00000100:00080000:0.0:1775765654.151201:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765654.151204:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765654.151213:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765654.151225:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741882368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765654.151235:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765654.151238:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765654.151240:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765654.151246:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1542 00000100:00080000:0.0:1775765654.151254:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765654.151255:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741882240:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765654.151262:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765654.151279:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765654.151287:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741882496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765654.151296:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765654.151316:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765654.151318:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765654.151323:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1542 00000100:00080000:0.0:1775765654.151326:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765654.151330:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765654.151337:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765654.151349:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741882624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765654.151395:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765654.151474:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765654/real 1775765654] req@ffff90c20363b800 x1862023741882240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765654.151497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741882240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765654.151510:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765654.151532:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765654.151538:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741882240:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765654.151582:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882368:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765654.151613:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765654.151629:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765654/real 1775765654] req@ffff90c20363b480 x1862023741882368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765654.151645:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741882368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765654.151656:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765654.151660:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765654.151664:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765654.151672:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882496:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765654.151707:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765654.151718:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765654/real 1775765654] req@ffff90c203638e00 x1862023741882496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765654.151730:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741882496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765654.151739:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765654.151751:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765654.151754:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765654.151778:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882624:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765654.151800:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765654.151811:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765654/real 1775765654] req@ffff90c203639f80 x1862023741882624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765654.151842:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741882624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765654.151853:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765654.151855:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765654.151858:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882624:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765655.175123:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765656.199215:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765657.223114:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765658.247144:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765659.271085:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765659.271096:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765659.271101:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765659.271112:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1547 00000100:00080000:3.0:1775765659.271125:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765659.271143:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765659.271174:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765659.271189:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741882752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765659.271220:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765659.271224:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765659.271226:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765659.271230:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1547 00000100:00080000:3.0:1775765659.271233:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765659.271236:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765659.271246:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765659.271259:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741882880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765659.271280:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765659.271283:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765659.271285:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765659.271289:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1547 00000100:00080000:3.0:1775765659.271292:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765659.271295:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765659.271299:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765659.271322:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741883008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765659.271335:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741882752:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765659.271336:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765659.271347:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765659.271349:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765659.271365:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1547 00000100:00080000:3.0:1775765659.271368:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765659.271370:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765659.271375:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765659.271388:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741883136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765659.271469:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765659.271534:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765659/real 1775765659] req@ffff90c218da0700 x1862023741882752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765659.271567:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741882752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765659.271586:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765659.271593:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765659.271600:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741882752:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765659.271641:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765659.271678:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765659.271698:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765659/real 1775765659] req@ffff90c218da3480 x1862023741882880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765659.271738:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741882880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765659.271759:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765659.271765:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765659.271770:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741882880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765659.271782:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765659.271817:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765659.271835:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765659/real 1775765659] req@ffff90c218da0380 x1862023741883008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765659.271854:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741883008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765659.271868:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765659.271873:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765659.271879:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765659.271890:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883136:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765659.272014:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765659.272033:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765659/real 1775765659] req@ffff90c218da2d80 x1862023741883136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765659.272052:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741883136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765659.272068:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765659.272074:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765659.272079:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883136:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765660.295169:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765661.319126:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765662.343168:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765663.367121:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765664.390982:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765664.390989:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765664.390993:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765664.391001:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1552 00000100:00080000:3.0:1775765664.391011:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765664.391018:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765664.391039:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765664.391053:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741883264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765664.391076:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765664.391084:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765664.391103:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765664.391107:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1552 00000100:00080000:3.0:1775765664.391111:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765664.391114:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765664.391122:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765664.391129:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741883264:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765664.391137:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741883392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765664.391146:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765664.391148:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765664.391150:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765664.391153:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1552 00000100:00080000:3.0:1775765664.391157:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765664.391160:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765664.391163:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765664.391171:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741883520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765664.391179:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765664.391181:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765664.391182:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765664.391185:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1552 00000100:00080000:3.0:1775765664.391208:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765664.391212:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765664.391219:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765664.391227:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765664.391244:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741883648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765664.391305:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765664/real 1775765664] req@ffff90c218da3800 x1862023741883264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765664.391320:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741883264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765664.391327:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765664.391330:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765664.391333:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741883264:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765664.391352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883392:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765664.391377:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765664.391384:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765664/real 1775765664] req@ffff90c218da1880 x1862023741883392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765664.391393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741883392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765664.391398:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765664.391400:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765664.391402:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765664.391405:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883520:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765664.391415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765664.391420:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765664/real 1775765664] req@ffff90c218da0e00 x1862023741883520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765664.391428:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741883520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765664.391432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765664.391434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765664.391436:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765664.391439:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883648:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765664.391448:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765664.391453:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765664/real 1775765664] req@ffff90c218da2d80 x1862023741883648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765664.391460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741883648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765664.391471:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765664.391473:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765664.391475:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883648:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765665.415163:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765666.439191:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765667.463160:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765668.487217:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765669.511076:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765669.511088:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765669.511093:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765669.511104:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1557 00000100:00080000:3.0:1775765669.511116:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765669.511125:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765669.511181:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741883776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765669.511243:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:0.0:1775765669.511244:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765669.511256:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765669.511261:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765669.511273:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1557 00000100:00080000:3.0:1775765669.511284:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765669.511292:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765669.511308:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765669.511328:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741883776:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765669.511330:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741883904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765669.511369:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765669.511373:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765669.511375:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765669.511380:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1557 00000100:00080000:3.0:1775765669.511385:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765669.511390:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765669.511394:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765669.511407:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741884032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765669.511418:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765669.511422:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765669.511424:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765669.511429:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1557 00000100:00080000:3.0:1775765669.511432:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765669.511436:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765669.511442:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765669.511455:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741884160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765669.511492:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765669.511569:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765669/real 1775765669] req@ffff90c218da0380 x1862023741883776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765669.511597:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741883776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765669.511637:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765669.511642:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765669.511646:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741883776:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765669.511681:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883904:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765669.511716:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765669.511728:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765669/real 1775765669] req@ffff90c218da3480 x1862023741883904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765669.511739:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741883904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765669.511748:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765669.511751:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765669.511754:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741883904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765669.511760:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765669.511776:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765669.511784:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765669/real 1775765669] req@ffff90c218da0700 x1862023741884032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765669.511795:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741884032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765669.511802:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765669.511805:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765669.511807:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765669.511813:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884160:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765669.511828:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765669.511836:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765669/real 1775765669] req@ffff90c218da3100 x1862023741884160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765669.511845:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741884160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765669.511855:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765669.511859:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765669.511861:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884160:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765670.535180:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765671.559241:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765672.583103:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765673.607195:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765674.631086:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775765674.631098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765674.631103:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765674.631112:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765674.631128:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1563 00000100:00080000:1.0:1775765674.631146:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765674.631156:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765674.631219:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741884288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765674.631263:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765674.631268:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765674.631271:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765674.631276:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1563 00000100:00080000:1.0:1775765674.631280:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765674.631285:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765674.631301:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765674.631325:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741884416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765674.631327:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741884288:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765674.631354:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765674.631359:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765674.631361:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765674.631366:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1563 00000100:00080000:1.0:1775765674.631370:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765674.631374:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765674.631380:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765674.631395:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741884544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765674.631407:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765674.631411:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765674.631414:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765674.631420:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1563 00000100:00080000:1.0:1775765674.631424:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765674.631429:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765674.631437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775765674.631442:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765674.631460:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741884672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765674.631500:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765674/real 1775765674] req@ffff90c203494e00 x1862023741884288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765674.631528:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741884288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765674.631543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765674.631548:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765674.631554:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741884288:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765674.631586:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765674.631621:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765674.631642:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765674/real 1775765674] req@ffff90c203497480 x1862023741884416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765674.631667:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741884416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765674.631685:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765674.631692:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765674.631700:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884416:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765674.631711:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884544:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765674.631747:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765674.631781:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765674/real 1775765674] req@ffff90c203494000 x1862023741884544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765674.631802:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741884544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765674.631845:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765674.631850:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765674.631854:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765674.631863:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884672:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765674.631925:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765674.631942:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765674/real 1775765674] req@ffff90c203495f80 x1862023741884672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765699 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765674.631966:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741884672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765699 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765674.631986:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765674.631992:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765674.631999:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884672:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765675.655134:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765676.679135:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765677.703188:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765678.727140:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765679.751026:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765679.751041:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765679.751048:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765679.751064:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1568 00000100:00080000:1.0:1775765679.751083:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765679.751094:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765679.751140:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765679.751162:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741884800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765679.751202:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765679.751208:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765679.751210:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765679.751216:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1568 00000100:00080000:1.0:1775765679.751220:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765679.751225:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765679.751239:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765679.751256:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741884928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765679.751274:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765679.751274:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741884800:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765679.751286:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765679.751291:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765679.751297:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1568 00000100:00080000:1.0:1775765679.751300:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765679.751305:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765679.751310:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765679.751326:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741885056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765679.751340:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765679.751345:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765679.751348:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765679.751352:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1568 00000100:00080000:1.0:1775765679.751356:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765679.751376:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765679.751387:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775765679.751388:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765679.751424:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741885184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765679.751463:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765679/real 1775765679] req@ffff90c203497100 x1862023741884800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765679.751485:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741884800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765679.751497:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765679.751502:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765679.751506:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741884800:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765679.751534:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884928:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765679.751556:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765679.751567:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765679/real 1775765679] req@ffff90c203496d80 x1862023741884928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765679.751579:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741884928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765679.751587:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765679.751590:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765679.751594:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741884928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765679.751598:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885056:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765679.751612:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765679.751620:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765679/real 1775765679] req@ffff90c203494380 x1862023741885056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765679.751630:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741885056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765679.751638:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765679.751640:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765679.751642:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765679.751647:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885184:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765679.751661:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765679.751668:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765679/real 1775765679] req@ffff90c203495c00 x1862023741885184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765704 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765679.751680:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741885184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765704 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765679.751690:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765679.751692:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765679.751695:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885184:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765680.775058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765681.799123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765682.823070:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765683.847011:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765684.871134:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765684.871143:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765684.871152:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765684.871157:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765684.871168:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1573 00000100:00080000:0.0:1775765684.871178:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765684.871183:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765684.871218:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741885312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765684.871247:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765684.871249:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765684.871251:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765684.871253:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1573 00000100:00080000:0.0:1775765684.871256:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765684.871258:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765684.871266:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765684.871278:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741885440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765684.871287:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765684.871290:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765684.871291:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765684.871293:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1573 00000100:00080000:0.0:1775765684.871296:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765684.871298:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765684.871302:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765684.871308:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741885568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765684.871319:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765684.871322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741885312:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765684.871323:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765684.871326:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765684.871348:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1573 00000100:00080000:0.0:1775765684.871353:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765684.871357:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765684.871364:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765684.871383:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741885696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765684.871453:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765684.871528:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765684/real 1775765684] req@ffff90c203638380 x1862023741885312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765684.871558:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741885312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765684.871574:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765684.871580:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765684.871585:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741885312:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765684.871597:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765684.871630:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765684.871645:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765684/real 1775765684] req@ffff90c20363a680 x1862023741885440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765684.871688:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741885440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765684.871708:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765684.871713:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765684.871745:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765684.871796:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765684.871825:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765684.871841:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765684/real 1775765684] req@ffff90c20363bb80 x1862023741885568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765684.871865:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741885568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765684.871926:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765684.871934:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765684.871942:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765684.871954:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885696:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765684.871993:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765684.872009:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765684/real 1775765684] req@ffff90c20363b100 x1862023741885696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765684.872025:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741885696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765709 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765684.872049:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765684.872053:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765684.872058:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885696:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765685.895047:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765686.919082:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765687.943111:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765688.967015:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765689.990979:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765689.990987:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765689.990992:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765689.991001:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1578 00000100:00080000:0.0:1775765689.991010:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765689.991016:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765689.991038:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765689.991051:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741885824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765689.991074:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765689.991076:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765689.991078:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765689.991080:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1578 00000100:00080000:0.0:1775765689.991083:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765689.991085:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765689.991093:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765689.991103:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741885952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765689.991107:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741885824:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765689.991115:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765689.991118:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765689.991119:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765689.991122:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1578 00000100:00080000:0.0:1775765689.991124:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765689.991126:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765689.991129:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765689.991136:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741886080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765689.991148:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765689.991150:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765689.991151:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765689.991153:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1578 00000100:00080000:0.0:1775765689.991155:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765689.991157:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765689.991161:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765689.991170:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741886208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765689.991171:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765689.991207:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765689/real 1775765689] req@ffff90c20363aa00 x1862023741885824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765689.991221:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741885824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765689.991228:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765689.991232:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765689.991235:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741885824:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765689.991251:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765689.991264:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765689.991271:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765689/real 1775765689] req@ffff90c203638000 x1862023741885952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765689.991279:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741885952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765689.991285:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765689.991287:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765689.991289:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741885952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765689.991292:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886080:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765689.991302:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765689.991308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765689/real 1775765689] req@ffff90c203639500 x1862023741886080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765689.991324:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741886080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765689.991330:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765689.991332:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765689.991334:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765689.991337:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886208:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765689.991348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765689.991354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765689/real 1775765689] req@ffff90c203639180 x1862023741886208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765689.991362:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741886208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765714 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765689.991368:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765689.991369:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765689.991371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886208:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765691.015117:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765692.039001:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765693.063086:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765694.087002:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765695.110984:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765695.110994:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765695.111000:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765695.111010:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1583 00000100:00080000:0.0:1775765695.111021:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765695.111028:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765695.111052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765695.111072:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741886336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765695.111100:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765695.111103:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765695.111105:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765695.111109:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1583 00000100:00080000:0.0:1775765695.111114:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765695.111119:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765695.111129:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765695.111138:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741886336:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765695.111146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741886464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765695.111159:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765695.111164:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765695.111166:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765695.111171:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1583 00000100:00080000:0.0:1775765695.111175:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765695.111179:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765695.111183:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765695.111196:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741886592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765695.111207:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765695.111211:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765695.111213:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765695.111217:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1583 00000100:00080000:0.0:1775765695.111222:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765695.111227:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765695.111233:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765695.111239:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765695.111248:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741886720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765695.111272:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765695.111298:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886720:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765696.135039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765697.159089:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765698.183136:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775765698.183137:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:2.0:1775765698.183176:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775765698.183190:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:2.0:1775765698.183227:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:2.0:1775765698.183235:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:2.0:1775765698.183243:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775765698.183287:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765695/real 1775765698] req@ffff90c203638000 x1862023741886720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765698.183316:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741886720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765698.183331:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765698.183336:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765698.183341:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886720:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775765698.183349:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765695/real 1775765698] req@ffff90c203639180 x1862023741886464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765698.183361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741886464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765698.183368:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765698.183371:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765698.183374:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765698.183380:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765695/real 1775765698] req@ffff90c203639500 x1862023741886592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765698.183391:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741886592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765698.183398:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765698.183401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765698.183405:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765698.183411:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765695/real 1775765698] req@ffff90c203638700 x1862023741886336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765698.183423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741886336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765698.183431:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765698.183434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765698.183437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741886336:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775765699.207038:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765700.231018:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765700.231028:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765700.231034:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765700.231043:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1588 00000100:00080000:0.0:1775765700.231054:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765700.231061:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765700.231088:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765700.231101:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741886848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765700.231129:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765700.231133:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765700.231136:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765700.231140:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1588 00000100:00080000:0.0:1775765700.231144:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765700.231147:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765700.231157:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765700.231163:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741886848:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765700.231173:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741886976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765700.231185:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765700.231189:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765700.231191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765700.231195:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1588 00000100:00080000:0.0:1775765700.231199:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765700.231203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765700.231208:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765700.231220:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765700.231221:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741887104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765700.231233:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765700.231237:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765700.231240:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765700.231244:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1588 00000100:00080000:0.0:1775765700.231248:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765700.231251:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765700.231257:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765700.231261:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765700/real 1775765700] req@ffff90c203638700 x1862023741886848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765700.231270:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741887232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765700.231282:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741886848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765700.231295:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765700.231300:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765700.231303:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741886848:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765700.231326:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765700.231353:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765700.231366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765700/real 1775765700] req@ffff90c203639500 x1862023741886976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765700.231379:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741886976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765700.231387:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765700.231392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765700.231395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741886976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765700.231401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765700.231422:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765700.231433:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765700/real 1775765700] req@ffff90c203639180 x1862023741887104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765700.231444:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741887104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765700.231452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765700.231463:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765700.231466:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765700.231472:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887232:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765700.231493:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765700.231505:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765700/real 1775765700] req@ffff90c203638000 x1862023741887232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765700.231517:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741887232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765700.231528:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765700.231532:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765700.231534:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887232:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765701.255042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765702.279050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765703.303060:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765704.326977:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765705.350960:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765705.350979:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765705.350984:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765705.350996:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1593 00000100:00080000:1.0:1775765705.351011:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765705.351018:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765705.351030:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765705.351059:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741887360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765705.351081:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765705.351084:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765705.351086:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765705.351089:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1593 00000100:00080000:1.0:1775765705.351092:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765705.351095:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765705.351103:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765705.351113:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741887488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765705.351121:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765705.351124:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741887360:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765705.351125:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765705.351127:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765705.351133:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1593 00000100:00080000:1.0:1775765705.351137:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765705.351139:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765705.351142:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765705.351151:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741887616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765705.351157:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765705.351159:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765705.351160:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765705.351163:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1593 00000100:00080000:1.0:1775765705.351165:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765705.351167:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765705.351170:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765705.351178:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741887744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765705.351207:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765705.351245:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765705/real 1775765705] req@ffff90c203495880 x1862023741887360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765705.351266:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741887360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765705.351276:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765705.351281:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765705.351285:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741887360:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765705.351309:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887488:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765705.351333:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765705.351346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765705/real 1775765705] req@ffff90c203497b80 x1862023741887488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765705.351360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741887488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765705.351369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765705.351372:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765705.351376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765705.351382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887616:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765705.351402:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765705.351413:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765705/real 1775765705] req@ffff90c203496300 x1862023741887616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765705.351432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741887616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765705.351439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765705.351443:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765705.351446:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765705.351453:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887744:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765705.351474:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765705.351484:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765705/real 1775765705] req@ffff90c203496a00 x1862023741887744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765705.351497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741887744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765705.351508:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765705.351511:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765705.351514:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741887744:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765706.374983:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765707.399018:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765708.422991:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765709.447045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765710.470964:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765710.470971:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765710.470976:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765710.470983:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1598 00000100:00080000:1.0:1775765710.470993:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765710.470999:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765710.471034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765710.471035:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741887872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765710.471058:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765710.471061:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765710.471063:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765710.471066:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1598 00000100:00080000:1.0:1775765710.471068:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765710.471071:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765710.471079:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765710.471089:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741888000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765710.471090:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741887872:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765710.471105:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765710.471107:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765710.471108:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765710.471111:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1598 00000100:00080000:1.0:1775765710.471114:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765710.471116:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765710.471120:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765710.471128:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741888128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765710.471134:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765710.471137:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765710.471138:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765710.471141:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1598 00000100:00080000:1.0:1775765710.471143:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765710.471145:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765710.471148:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765710.471153:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765710.471159:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741888256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765710.471185:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765710/real 1775765710] req@ffff90c203496680 x1862023741887872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765710.471199:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741887872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765710.471212:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765710.471215:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765710.471218:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741887872:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765710.471235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765710.471248:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765710.471255:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765710/real 1775765710] req@ffff90c203494a80 x1862023741888000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765710.471262:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741888000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765710.471268:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765710.471270:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765710.471272:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765710.471276:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765710.471284:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765710.471289:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765710/real 1775765710] req@ffff90c203495180 x1862023741888128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765710.471296:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741888128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765710.471301:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765710.471303:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765710.471304:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765710.471308:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888256:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765710.471318:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765710.471323:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765710/real 1775765710] req@ffff90c203495500 x1862023741888256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765710.471330:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741888256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765710.471336:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765710.471338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765710.471339:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888256:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765711.495009:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765712.519052:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765713.543123:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765714.567155:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765715.591083:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765715.591094:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765715.591100:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765715.591110:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1604 00000100:00080000:1.0:1775765715.591123:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765715.591130:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765715.591199:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741888384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765715.591216:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765715.591284:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765715.591291:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765715.591294:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765715.591301:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1604 00000100:00080000:1.0:1775765715.591308:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765715.591314:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765715.591327:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765715.591349:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741888512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765715.591385:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741888384:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765715.591397:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765715.591402:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765715.591404:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765715.591409:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1604 00000100:00080000:1.0:1775765715.591414:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765715.591420:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765715.591489:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765715.591520:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741888640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765715.591531:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765715.591534:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765715.591535:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765715.591551:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1604 00000100:00080000:1.0:1775765715.591557:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765715.591563:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765715.591569:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765715.591587:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741888768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765715.591592:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765715.591671:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765715/real 1775765715] req@ffff90c203494700 x1862023741888384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765715.591711:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741888384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765715.591726:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765715.591733:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765715.591739:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741888384:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765715.591787:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765715.591828:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765715.591842:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765715/real 1775765715] req@ffff90c203495500 x1862023741888512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765715.591859:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741888512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765715.591870:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765715.591873:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765715.591877:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765715.591885:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765715.591956:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765715.591971:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765715/real 1775765715] req@ffff90c203495180 x1862023741888640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765715.591986:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741888640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765715.591996:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765715.592001:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765715.592004:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765715.592012:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888768:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765715.592034:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765715.592044:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765715/real 1775765715] req@ffff90c203494a80 x1862023741888768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765715.592058:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741888768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765715.592079:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765715.592084:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765715.592088:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741888768:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765716.615140:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765717.639148:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765718.663084:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765719.687042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765720.711041:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765720.711054:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765720.711061:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765720.711075:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1609 00000100:00080000:1.0:1775765720.711090:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765720.711101:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765720.711137:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765720.711161:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741888896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765720.711192:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765720.711196:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765720.711198:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765720.711202:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1609 00000100:00080000:1.0:1775765720.711206:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765720.711210:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765720.711219:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765720.711231:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741889024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765720.711242:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765720.711246:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765720.711247:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765720.711251:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1609 00000100:00080000:1.0:1775765720.711253:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765720.711257:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765720.711261:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765720.711270:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741888896:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765720.711272:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741889152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765720.711291:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765720.711294:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765720.711295:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765720.711298:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1609 00000100:00080000:1.0:1775765720.711301:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765720.711304:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765720.711308:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765720.711318:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741889280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765720.711368:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765720.711448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765720/real 1775765720] req@ffff90c203496680 x1862023741888896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765720.711468:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741888896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765720.711478:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765720.711482:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765720.711485:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741888896:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765720.711493:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889024:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765720.711521:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765720.711531:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765720/real 1775765720] req@ffff90c203496a00 x1862023741889024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765720.711541:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741889024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765720.711548:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765720.711551:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765720.711554:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765720.711577:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889152:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765720.711590:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765720.711598:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765720/real 1775765720] req@ffff90c203496300 x1862023741889152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765720.711608:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741889152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765720.711615:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765720.711617:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765720.711620:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765720.711625:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889280:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765720.711637:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765720.711644:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765720/real 1775765720] req@ffff90c203497b80 x1862023741889280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765745 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765720.711653:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741889280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765745 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765720.711662:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765720.711664:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765720.711667:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889280:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765721.735107:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765722.759111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765723.783119:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765724.807031:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765725.830951:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765725.830959:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765725.830963:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765725.830972:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1614 00000100:00080000:1.0:1775765725.830992:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765725.830999:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765725.831019:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765725.831032:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741889408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765725.831054:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765725.831056:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765725.831058:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765725.831061:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1614 00000100:00080000:1.0:1775765725.831063:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765725.831065:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765725.831074:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765725.831084:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741889536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765725.831088:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741889408:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765725.831100:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765725.831103:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765725.831104:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765725.831107:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1614 00000100:00080000:1.0:1775765725.831108:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765725.831111:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765725.831114:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765725.831121:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741889664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765725.831127:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765725.831129:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765725.831130:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765725.831133:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1614 00000100:00080000:1.0:1775765725.831136:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765725.831138:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765725.831142:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765725.831149:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741889792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765725.831158:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765725.831194:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765725/real 1775765725] req@ffff90c203495880 x1862023741889408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765725.831208:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741889408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765725.831216:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765725.831220:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765725.831223:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741889408:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765725.831240:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889536:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765725.831255:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765725.831263:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765725/real 1775765725] req@ffff90c203495c00 x1862023741889536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765725.831271:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741889536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765725.831276:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765725.831278:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765725.831280:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765725.831284:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765725.831293:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765725.831298:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765725/real 1775765725] req@ffff90c203494380 x1862023741889664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765725.831305:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741889664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765725.831315:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765725.831317:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765725.831319:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765725.831322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889792:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765725.831363:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765725.831371:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765725/real 1775765725] req@ffff90c203496d80 x1862023741889792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765725.831378:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741889792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765750 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765725.831384:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765725.831386:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765725.831388:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741889792:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765726.855043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765727.879174:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765728.903059:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765729.927130:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765730.951038:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765730.951048:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765730.951053:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765730.951064:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1619 00000100:00080000:3.0:1775765730.951082:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765730.951090:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765730.951125:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765730.951134:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741889920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765730.951164:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765730.951167:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765730.951169:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765730.951173:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1619 00000100:00080000:3.0:1775765730.951176:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765730.951179:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765730.951189:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765730.951201:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741890048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765730.951212:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765730.951216:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765730.951217:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765730.951222:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1619 00000100:00080000:3.0:1775765730.951225:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765730.951233:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741889920:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765730.951233:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765730.951240:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765730.951251:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741890176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765730.951260:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765730.951264:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765730.951265:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765730.951269:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1619 00000100:00080000:3.0:1775765730.951274:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765730.951303:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765730.951311:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765730.951330:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741890304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765730.951343:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765730.951390:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765730/real 1775765730] req@ffff90c218da1f80 x1862023741889920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765730.951410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741889920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765730.951420:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765730.951425:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765730.951429:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741889920:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765730.951437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890048:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765730.951457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765730.951468:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765730/real 1775765730] req@ffff90c218da0000 x1862023741890048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765730.951479:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741890048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765730.951486:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765730.951489:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765730.951492:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765730.951515:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890176:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765730.951530:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765730.951538:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765730/real 1775765730] req@ffff90c218da0a80 x1862023741890176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765730.951548:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741890176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765730.951555:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765730.951558:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765730.951561:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765730.951566:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890304:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765730.951580:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765730.951587:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765730/real 1775765730] req@ffff90c218da3100 x1862023741890304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765730.951597:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741890304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765755 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765730.951606:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765730.951609:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765730.951611:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890304:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765731.975096:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765732.999084:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765734.023187:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765735.047179:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765736.071002:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765736.071013:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765736.071019:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765736.071030:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1624 00000100:00080000:3.0:1775765736.071042:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765736.071050:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765736.071093:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741890432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765736.071111:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765736.071121:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765736.071125:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765736.071127:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765736.071131:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1624 00000100:00080000:3.0:1775765736.071134:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765736.071138:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765736.071146:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765736.071156:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741890560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765736.071184:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765736.071186:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741890432:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765736.071190:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765736.071197:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765736.071205:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1624 00000100:00080000:3.0:1775765736.071209:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765736.071211:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765736.071215:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765736.071265:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741890688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765736.071274:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765736.071276:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765736.071279:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765736.071283:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1624 00000100:00080000:3.0:1775765736.071305:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765736.071308:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765736.071314:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765736.071320:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765736.071328:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741890816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765736.071383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765736/real 1775765736] req@ffff90c218da0700 x1862023741890432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765736.071405:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741890432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765736.071417:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765736.071421:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765736.071425:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741890432:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765736.071456:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890560:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765736.071476:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765736.071486:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765736/real 1775765736] req@ffff90c218da3480 x1862023741890560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765736.071497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741890560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765736.071510:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765736.071513:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765736.071517:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765736.071521:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765736.071564:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765736.071576:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765736/real 1775765736] req@ffff90c218da0380 x1862023741890688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765736.071590:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741890688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765736.071600:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765736.071603:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765736.071607:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765736.071614:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890816:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765736.071634:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765736.071643:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765736/real 1775765736] req@ffff90c218da2d80 x1862023741890816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765736.071657:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741890816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765736.071669:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765736.071672:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765736.071677:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741890816:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765737.095162:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765738.119140:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765739.143096:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765740.167252:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765741.191040:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765741.191052:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765741.191059:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765741.191072:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1629 00000100:00080000:0.0:1775765741.191086:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765741.191097:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765741.191121:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765741.191166:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741890944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765741.191233:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765741.191239:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765741.191243:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765741.191251:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1629 00000100:00080000:0.0:1775765741.191258:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765741.191264:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765741.191277:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765741.191302:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741891072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765741.191311:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741890944:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765741.191322:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765741.191330:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765741.191332:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765741.191353:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1629 00000100:00080000:0.0:1775765741.191358:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765741.191362:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765741.191370:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765741.191385:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741891200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765741.191397:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765741.191402:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765741.191404:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765741.191408:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1629 00000100:00080000:0.0:1775765741.191412:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765741.191417:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765741.191424:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765741.191425:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765741.191442:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741891328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765741.191516:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765741/real 1775765741] req@ffff90c20363aa00 x1862023741890944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765741.191569:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741890944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765741.191600:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765741.191609:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765741.191616:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741890944:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765741.191649:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765741.191683:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765741.191699:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765741/real 1775765741] req@ffff90c20363b100 x1862023741891072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765741.191715:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741891072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765741.191725:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765741.191729:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765741.191733:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765741.191740:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765741.191758:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765741.191769:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765741/real 1775765741] req@ffff90c20363bb80 x1862023741891200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765741.191782:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741891200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765741.191791:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765741.191794:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765741.191797:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765741.191803:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891328:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765741.191820:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765741.191829:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765741/real 1775765741] req@ffff90c20363a680 x1862023741891328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765741.191841:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741891328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765741.191852:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765741.191856:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765741.191859:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891328:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765742.215065:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765743.239070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765744.263033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765745.287212:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765746.311047:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765746.311062:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765746.311069:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765746.311083:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1634 00010000:00010000:0.0:1775765746.311084:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765746.311101:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765746.311113:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765746.311195:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741891456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765746.311229:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765746.311234:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765746.311236:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765746.311240:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1634 00000100:00080000:1.0:1775765746.311245:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765746.311248:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765746.311258:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765746.311272:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741891584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765746.311284:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765746.311291:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775765746.311293:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741891456:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765746.311294:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765746.311308:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1634 00000100:00080000:1.0:1775765746.311311:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765746.311315:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765746.311319:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765746.311333:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741891712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765746.311354:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765746.311358:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765746.311360:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765746.311364:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1634 00000100:00080000:1.0:1775765746.311367:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765746.311371:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765746.311376:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765746.311387:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765746.311393:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741891840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765746.311452:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765746/real 1775765746] req@ffff90c203497100 x1862023741891456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765746.311475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741891456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765746.311521:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765746.311529:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765746.311535:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741891456:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765746.311568:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891584:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765746.311606:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765746.311622:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765746/real 1775765746] req@ffff90c203495f80 x1862023741891584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765746.311636:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741891584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765746.311644:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765746.311648:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765746.311651:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765746.311658:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891712:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765746.311675:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765746.311684:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765746/real 1775765746] req@ffff90c203494000 x1862023741891712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765746.311695:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741891712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765746.311703:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765746.311705:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765746.311708:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765746.311714:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891840:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765746.311729:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765746.311737:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765746/real 1775765746] req@ffff90c203497480 x1862023741891840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765746.311748:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741891840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765746.311757:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765746.311760:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765746.311763:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741891840:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765747.335240:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765748.359066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765749.383295:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765750.407205:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765751.430969:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765751.430991:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765751.430996:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765751.431007:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1639 00000100:00080000:1.0:1775765751.431020:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765751.431027:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765751.431071:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741891968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765751.431116:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765751.431120:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765751.431123:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765751.431132:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1639 00000100:00080000:1.0:1775765751.431139:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765751.431145:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765751.431157:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00100000:1.0:1775765751.431158:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765751.431177:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741892096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765751.431192:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741891968:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765751.431196:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765751.431217:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765751.431219:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765751.431224:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1639 00000100:00080000:1.0:1775765751.431236:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765751.431241:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765751.431247:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765751.431263:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741892224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765751.431273:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765751.431277:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765751.431279:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765751.431283:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1639 00000100:00080000:1.0:1775765751.431287:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765751.431291:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765751.431295:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765751.431305:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741892352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765751.431321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765751.431379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765751/real 1775765751] req@ffff90c203494e00 x1862023741891968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765751.431403:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741891968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765751.431416:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765751.431421:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765751.431426:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741891968:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765751.431458:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765751.431481:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765751.431493:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765751/real 1775765751] req@ffff90c203497480 x1862023741892096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765751.431512:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741892096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765751.431526:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765751.431532:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765751.431537:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765751.431548:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765751.431581:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765751.431598:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765751/real 1775765751] req@ffff90c203494000 x1862023741892224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765751.431616:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741892224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765751.431625:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765751.431629:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765751.431632:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765751.431638:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892352:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765751.431662:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765751.431672:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765751/real 1775765751] req@ffff90c203495f80 x1862023741892352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765751.431684:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741892352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765751.431702:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765751.431706:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765751.431709:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892352:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765752.455146:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765753.479195:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765754.503144:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765755.527167:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765756.550980:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765756.550994:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765756.550998:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765756.551006:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1645 00000100:00080000:1.0:1775765756.551016:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775765756.551020:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765756.551021:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765756.551076:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741892480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765756.551098:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765756.551101:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765756.551102:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765756.551105:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1645 00000100:00080000:1.0:1775765756.551108:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765756.551110:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765756.551118:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765756.551127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741892608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765756.551134:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765756.551138:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765756.551141:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765756.551143:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741892480:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765756.551145:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1645 00000100:00080000:1.0:1775765756.551150:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765756.551154:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765756.551158:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765756.551166:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741892736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765756.551172:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765756.551174:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765756.551175:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765756.551178:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1645 00000100:00080000:1.0:1775765756.551180:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765756.551183:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765756.551185:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765756.551193:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741892864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765756.551218:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765756.551258:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765756/real 1775765756] req@ffff90c203497100 x1862023741892480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765756.551277:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741892480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765756.551288:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765756.551292:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765756.551297:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741892480:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765756.551320:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765756.551342:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765756.551354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765756/real 1775765756] req@ffff90c203496d80 x1862023741892608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765756.551367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741892608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765756.551376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765756.551380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765756.551383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765756.551390:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892736:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765756.551409:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765756.551420:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765756/real 1775765756] req@ffff90c203494380 x1862023741892736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765756.551431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741892736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765756.551439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765756.551442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765756.551445:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765756.551451:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892864:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765756.551470:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765756.551480:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765756/real 1775765756] req@ffff90c203495c00 x1862023741892864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765756.551491:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741892864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765756.551502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765756.551505:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765756.551509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741892864:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765757.575096:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765758.599075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765759.623112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765760.647044:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765761.671036:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765761.671047:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765761.671053:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765761.671063:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1650 00000100:00080000:3.0:1775765761.671077:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765761.671084:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765761.671131:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765761.671136:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741892992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765761.671162:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765761.671165:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765761.671166:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765761.671169:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1650 00000100:00080000:3.0:1775765761.671171:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765761.671175:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765761.671185:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765761.671199:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741893120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765761.671207:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741892992:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765761.671214:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765761.671219:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765761.671221:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765761.671227:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1650 00000100:00080000:3.0:1775765761.671231:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765761.671236:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765761.671243:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765761.671257:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741893248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765761.671265:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765761.671267:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765761.671269:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765761.671272:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1650 00000100:00080000:3.0:1775765761.671297:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765761.671301:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765761.671306:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765761.671321:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741893376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765761.671376:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765761.671416:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765761.671446:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893376:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765762.695085:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765763.719089:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765764.743163:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765764.743194:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775765764.743263:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765764.743282:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765764.743319:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765764.743333:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765764.743350:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775765764.743364:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765761/real 1775765764] req@ffff90c218da1880 x1862023741893120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765764.743391:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741893120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765764.743403:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765764.743408:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765764.743411:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765764.743416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765761/real 1775765764] req@ffff90c218da3800 x1862023741893248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765764.743425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741893248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765764.743430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765764.743433:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765764.743437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765764.743444:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765761/real 1775765764] req@ffff90c218da2d80 x1862023741893376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765764.743456:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741893376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765764.743474:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765764.743478:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765764.743481:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893376:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775765764.743489:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765761/real 1775765764] req@ffff90c218da0e00 x1862023741892992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765786 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765764.743503:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741892992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765786 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765764.743510:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765764.743513:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765764.743516:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741892992:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775765765.767060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765766.791067:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765766.791069:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765766.791079:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765766.791084:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765766.791093:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1655 00000100:00080000:1.0:1775765766.791104:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765766.791110:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765766.791145:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741893504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765766.791169:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765766.791171:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765766.791173:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765766.791176:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1655 00000100:00080000:1.0:1775765766.791178:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765766.791180:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765766.791188:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765766.791196:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741893632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765766.791204:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765766.791207:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765766.791208:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765766.791210:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1655 00000100:00080000:1.0:1775765766.791213:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765766.791216:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765766.791219:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765766.791225:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741893760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765766.791235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741893504:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765766.791238:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765766.791249:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765766.791256:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765766.791260:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1655 00000100:00080000:1.0:1775765766.791262:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765766.791264:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765766.791267:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765766.791274:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741893888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765766.791360:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765766.791414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765766/real 1775765766] req@ffff90c203495880 x1862023741893504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765766.791440:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741893504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765766.791462:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765766.791469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765766.791474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741893504:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765766.791485:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765766.791519:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765766.791534:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765766/real 1775765766] req@ffff90c203497b80 x1862023741893632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765766.791548:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741893632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765766.791557:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765766.791562:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765766.791566:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765766.791605:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765766.791634:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765766.791645:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765766/real 1775765766] req@ffff90c203496300 x1862023741893760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765766.791657:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741893760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765766.791666:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765766.791671:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765766.791674:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765766.791682:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893888:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765766.791706:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765766.791718:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765766/real 1775765766] req@ffff90c203496a00 x1862023741893888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765791 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765766.791739:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741893888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765791 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765766.791752:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765766.791756:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765766.791759:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741893888:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765767.815194:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765768.839135:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765769.863142:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765770.887084:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765771.911088:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765771.911105:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765771.911112:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765771.911130:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1660 00010000:00010000:3.0:1775765771.911138:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765771.911149:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765771.911162:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765771.911222:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741894016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765771.911268:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765771.911274:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765771.911276:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765771.911283:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1660 00000100:00080000:0.0:1775765771.911290:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765771.911298:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765771.911312:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765771.911337:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741894144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765771.911339:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741894016:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765771.911368:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765771.911374:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765771.911376:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765771.911383:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1660 00000100:00080000:0.0:1775765771.911388:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765771.911394:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765771.911417:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765771.911440:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741894272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765771.911456:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765771.911462:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1775765771.911468:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765771.911470:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765771.911482:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1660 00000100:00080000:0.0:1775765771.911492:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765771.911499:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765771.911511:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765771.911519:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765771/real 1775765771] req@ffff90c203638380 x1862023741894016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765771.911542:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741894400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765771.911545:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741894016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765771.911577:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765771.911584:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765771.911599:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741894016:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765771.911640:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765771.911667:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765771.911682:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765771/real 1775765771] req@ffff90c203639f80 x1862023741894144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765771.911713:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741894144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765771.911732:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765771.911736:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765771.911740:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765771.911749:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894272:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765771.911770:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765771.911781:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765771/real 1775765771] req@ffff90c203638e00 x1862023741894272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765771.911795:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741894272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765771.911815:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765771.911819:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765771.911822:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765771.911829:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894400:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765771.911846:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765771.911856:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765771/real 1775765771] req@ffff90c20363b480 x1862023741894400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765771.911868:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741894400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765796 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765771.911880:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765771.911949:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765771.911955:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894400:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765772.935020:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765773.959025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765774.983110:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765776.007084:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765777.031066:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765777.031076:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765777.031081:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:0.0:1775765777.031086:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765777.031092:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1665 00000100:00080000:3.0:1775765777.031105:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765777.031112:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765777.031151:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741894528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765777.031181:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765777.031185:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765777.031187:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765777.031193:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1665 00000100:00080000:3.0:1775765777.031197:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765777.031201:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765777.031212:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765777.031222:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741894528:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765777.031248:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741894656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765777.031267:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765777.031271:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765777.031273:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765777.031279:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1665 00000100:00080000:3.0:1775765777.031284:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765777.031290:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765777.031296:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765777.031313:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741894784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765777.031318:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765777.031332:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765777.031337:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765777.031339:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765777.031346:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1665 00000100:00080000:3.0:1775765777.031352:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765777.031357:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765777.031363:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765777.031369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765777/real 1775765777] req@ffff90c218da0380 x1862023741894528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765777.031384:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741894912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765777.031391:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741894528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765777.031452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765777.031458:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765777.031463:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741894528:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765777.031493:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765777.031513:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765777.031522:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765777/real 1775765777] req@ffff90c218da3480 x1862023741894656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765777.031533:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741894656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765777.031540:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765777.031543:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765777.031557:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765777.031561:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765777.031573:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765777.031580:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765777/real 1775765777] req@ffff90c218da0700 x1862023741894784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765777.031593:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741894784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765777.031603:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765777.031606:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765777.031611:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765777.031629:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894912:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765777.031681:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765777.031692:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765777/real 1775765777] req@ffff90c218da3100 x1862023741894912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765777.031703:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741894912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765777.031714:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765777.031718:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765777.031721:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741894912:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765778.055064:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765779.079088:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765780.103038:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765781.127116:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765782.151028:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765782.151041:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765782.151048:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765782.151061:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1670 00000100:00080000:1.0:1775765782.151076:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765782.151085:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765782.151104:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765782.151139:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741895040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765782.151172:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765782.151176:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765782.151178:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765782.151181:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1670 00000100:00080000:1.0:1775765782.151185:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765782.151188:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765782.151200:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765782.151212:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741895168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765782.151225:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765782.151227:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741895040:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765782.151234:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765782.151240:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765782.151245:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1670 00000100:00080000:1.0:1775765782.151248:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765782.151251:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765782.151255:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765782.151268:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741895296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765782.151276:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765782.151280:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765782.151281:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765782.151286:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1670 00000100:00080000:1.0:1775765782.151291:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765782.151293:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765782.151297:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765782.151306:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741895424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765782.151324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765782.151379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765782/real 1775765782] req@ffff90c203496680 x1862023741895040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765782.151408:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741895040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765782.151426:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765782.151432:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765782.151438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741895040:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765782.151471:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765782.151497:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765782.151509:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765782/real 1775765782] req@ffff90c203494a80 x1862023741895168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765782.151521:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741895168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765782.151529:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765782.151532:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765782.151535:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765782.151541:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765782.151555:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765782.151562:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765782/real 1775765782] req@ffff90c203495180 x1862023741895296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765782.151573:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741895296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765782.151580:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765782.151582:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765782.151585:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765782.151590:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895424:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765782.151604:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765782.151611:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765782/real 1775765782] req@ffff90c203495500 x1862023741895424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765782.151621:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741895424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765782.151630:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765782.151633:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765782.151635:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895424:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765783.175141:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765784.199132:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765785.223078:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765786.247181:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765787.270980:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765787.270987:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765787.270991:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765787.271001:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1675 00000100:00080000:1.0:1775765787.271010:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765787.271017:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765787.271052:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741895552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775765787.271058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765787.271075:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765787.271078:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765787.271079:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765787.271082:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1675 00000100:00080000:1.0:1775765787.271085:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765787.271088:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765787.271096:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765787.271108:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741895680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765787.271109:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741895552:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765787.271122:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765787.271124:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765787.271125:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765787.271128:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1675 00000100:00080000:1.0:1775765787.271131:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765787.271133:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765787.271136:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765787.271144:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741895808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765787.271150:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765787.271152:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765787.271153:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765787.271155:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1675 00000100:00080000:1.0:1775765787.271158:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765787.271161:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765787.271165:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765787.271172:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765787.271174:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741895936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765787.271207:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765787/real 1775765787] req@ffff90c203494700 x1862023741895552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765787.271221:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741895552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765787.271229:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765787.271232:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765787.271235:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741895552:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765787.271251:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765787.271265:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765787.271272:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765787/real 1775765787] req@ffff90c203495500 x1862023741895680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765787.271280:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741895680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765787.271285:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765787.271287:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765787.271289:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765787.271292:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895808:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765787.271301:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765787.271306:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765787/real 1775765787] req@ffff90c203495180 x1862023741895808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765787.271313:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741895808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765787.271318:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765787.271320:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765787.271322:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765787.271325:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895936:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765787.271334:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765787.271339:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765787/real 1775765787] req@ffff90c203494a80 x1862023741895936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765787.271345:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741895936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765787.271355:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765787.271357:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765787.271359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741895936:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765788.295095:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765789.319067:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765790.343132:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765791.367109:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765792.390996:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765792.391004:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765792.391008:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765792.391016:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1680 00000100:00080000:1.0:1775765792.391026:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765792.391032:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765792.391068:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741896064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765792.391075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765792.391092:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765792.391095:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765792.391097:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765792.391099:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1680 00000100:00080000:1.0:1775765792.391102:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765792.391105:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765792.391112:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765792.391123:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741896192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765792.391124:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741896064:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765792.391135:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765792.391138:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765792.391139:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765792.391142:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1680 00000100:00080000:1.0:1775765792.391144:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765792.391146:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765792.391150:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765792.391157:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741896320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765792.391164:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765792.391166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765792.391167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765792.391169:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1680 00000100:00080000:1.0:1775765792.391171:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765792.391173:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765792.391176:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765792.391184:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765792.391185:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741896448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765792.391220:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765792/real 1775765792] req@ffff90c203496680 x1862023741896064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765792.391234:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741896064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765792.391242:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765792.391245:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765792.391248:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741896064:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765792.391272:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765792.391287:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765792.391294:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765792/real 1775765792] req@ffff90c203496a00 x1862023741896192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765792.391302:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741896192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765792.391308:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765792.391310:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765792.391312:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765792.391316:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765792.391326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765792.391331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765792/real 1775765792] req@ffff90c203496300 x1862023741896320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765792.391338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741896320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765792.391343:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765792.391344:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765792.391346:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765792.391349:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896448:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765792.391359:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765792.391363:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765792/real 1775765792] req@ffff90c203497b80 x1862023741896448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765792.391369:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741896448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765792.391375:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765792.391377:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765792.391378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896448:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765793.415078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765794.439062:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765795.463020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765796.487070:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765797.511022:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765797.511032:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765797.511038:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765797.511048:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1685 00000100:00080000:3.0:1775765797.511059:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765797.511065:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765797.511092:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765797.511106:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741896576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765797.511137:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765797.511141:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765797.511142:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765797.511146:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1685 00000100:00080000:3.0:1775765797.511149:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765797.511152:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765797.511161:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765797.511172:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741896704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765797.511183:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765797.511188:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765797.511192:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765797.511194:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741896576:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765797.511199:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1685 00000100:00080000:3.0:1775765797.511210:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765797.511214:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765797.511218:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765797.511230:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741896832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765797.511238:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765797.511241:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765797.511243:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765797.511246:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1685 00000100:00080000:3.0:1775765797.511249:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765797.511252:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765797.511257:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765797.511267:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741896960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765797.511300:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765797.511354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765797/real 1775765797] req@ffff90c218da0a80 x1862023741896576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765797.511372:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741896576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765797.511382:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765797.511386:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765797.511390:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741896576:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765797.511414:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765797.511437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765797.511446:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765797/real 1775765797] req@ffff90c218da0000 x1862023741896704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765797.511455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741896704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765797.511461:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765797.511464:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765797.511467:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765797.511474:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896832:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765797.511496:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765797.511508:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765797/real 1775765797] req@ffff90c218da1f80 x1862023741896832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765797.511520:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741896832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765797.511533:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765797.511536:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765797.511540:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765797.511563:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896960:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765797.511600:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765797.511612:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765797/real 1775765797] req@ffff90c218da3100 x1862023741896960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765797.511625:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741896960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765797.511684:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765797.511687:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765797.511689:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741896960:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765798.535188:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765799.559077:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765800.583035:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765801.607182:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765802.631060:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765802.631075:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765802.631083:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775765802.631083:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765802.631102:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1691 00000100:00080000:0.0:1775765802.631121:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765802.631130:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765802.631191:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741897088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765802.631232:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765802.631237:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765802.631241:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765802.631248:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1691 00000100:00080000:0.0:1775765802.631254:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765802.631259:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765802.631277:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765802.631289:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741897088:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765802.631300:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741897216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765802.631317:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765802.631322:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765802.631325:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765802.631331:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1691 00000100:00080000:0.0:1775765802.631336:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765802.631342:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765802.631351:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765802.631365:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741897344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765802.631375:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765802.631392:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765802.631396:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765802.631399:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765802.631406:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1691 00000100:00080000:0.0:1775765802.631413:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765802.631419:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765802.631426:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765802.631431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765802/real 1775765802] req@ffff90c20363b800 x1862023741897088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765802.631447:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741897472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765802.631458:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741897088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765802.631496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765802.631502:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765802.631509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741897088:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765802.631533:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897216:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765802.631553:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765802.631561:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765802/real 1775765802] req@ffff90c20363b480 x1862023741897216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765802.631570:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741897216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765802.631575:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765802.631578:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765802.631580:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765802.631586:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765802.631619:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765802.631632:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765802/real 1775765802] req@ffff90c203638e00 x1862023741897344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765802.631645:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741897344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765802.631653:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765802.631657:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765802.631659:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765802.631664:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897472:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765802.631680:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765802.631687:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765802/real 1775765802] req@ffff90c203639f80 x1862023741897472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765827 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765802.631695:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741897472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765827 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765802.631702:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765802.631704:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765802.631705:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897472:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765803.655115:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765804.679069:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765805.703243:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765806.727111:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765807.751048:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765807.751056:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765807.751061:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765807.751071:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1696 00000100:00080000:0.0:1775765807.751082:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765807.751089:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765807.751103:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765807.751127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741897600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765807.751153:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765807.751155:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765807.751157:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765807.751159:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1696 00000100:00080000:0.0:1775765807.751162:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765807.751164:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765807.751171:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765807.751181:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741897728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765807.751188:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765807.751191:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765807.751193:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765807.751197:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1696 00000100:00080000:0.0:1775765807.751200:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765807.751203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765807.751208:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765807.751216:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741897600:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765807.751223:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741897856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765807.751240:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765807.751242:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765807.751243:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765807.751245:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1696 00000100:00080000:0.0:1775765807.751247:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765807.751251:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765807.751255:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765807.751266:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741897984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765807.751338:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765807.751424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765807/real 1775765807] req@ffff90c203638380 x1862023741897600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765807.751465:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741897600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765807.751485:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765807.751492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765807.751497:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741897600:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765807.751509:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765807.751542:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765807.751593:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765807/real 1775765807] req@ffff90c20363a680 x1862023741897728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765807.751609:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741897728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765807.751628:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765807.751633:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765807.751637:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765807.751673:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765807.751696:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765807.751707:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765807/real 1775765807] req@ffff90c20363bb80 x1862023741897856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765807.751721:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741897856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765807.751731:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765807.751735:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765807.751738:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765807.751745:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897984:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765807.751763:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765807.751774:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765807/real 1775765807] req@ffff90c20363b100 x1862023741897984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765832 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765807.751803:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741897984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765832 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765807.751813:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765807.751816:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765807.751820:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741897984:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765808.775097:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765809.799095:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765810.823165:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765811.847065:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765812.871018:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765812.871029:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765812.871035:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765812.871045:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1701 00010000:00010000:0.0:1775765812.871048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765812.871056:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765812.871063:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765812.871100:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741898112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765812.871128:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765812.871135:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765812.871137:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765812.871143:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1701 00000100:00080000:3.0:1775765812.871148:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765812.871153:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765812.871164:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741898112:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775765812.871164:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765812.871189:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741898240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765812.871203:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765812.871206:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765812.871207:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765812.871211:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1701 00000100:00080000:3.0:1775765812.871225:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765812.871229:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765812.871233:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775765812.871239:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765812.871248:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741898368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765812.871257:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765812.871260:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765812.871261:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765812.871265:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1701 00000100:00080000:3.0:1775765812.871268:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765812.871271:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765812.871277:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765812.871284:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765812/real 1775765812] req@ffff90c218da0700 x1862023741898112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765812.871295:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741898496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765812.871313:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741898112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765812.871338:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765812.871344:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765812.871349:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741898112:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765812.871373:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898240:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765812.871416:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765812.871431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765812/real 1775765812] req@ffff90c218da3480 x1862023741898240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765812.871447:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741898240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765812.871457:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765812.871461:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765812.871465:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765812.871470:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898368:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765812.871488:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765812.871532:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765812/real 1775765812] req@ffff90c218da0380 x1862023741898368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765812.871547:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741898368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765812.871557:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765812.871572:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765812.871577:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765812.871591:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898496:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765812.871610:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765812.871618:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765812/real 1775765812] req@ffff90c218da0e00 x1862023741898496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765812.871627:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741898496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765837 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765812.871634:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765812.871636:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765812.871639:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898496:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765813.895060:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765814.919121:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765815.943043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765816.967180:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765817.990994:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765817.991008:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765817.991014:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765817.991026:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1706 00000100:00080000:3.0:1775765817.991039:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765817.991046:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765817.991084:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741898624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765817.991115:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765817.991118:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765817.991120:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765817.991124:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1706 00000100:00080000:3.0:1775765817.991127:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765817.991132:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765817.991146:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765817.991155:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741898752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765817.991166:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765817.991171:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765817.991173:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765817.991178:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1706 00000100:00080000:3.0:1775765817.991187:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765817.991194:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765817.991200:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765817.991209:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741898880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765817.991216:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765817.991219:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765817.991220:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765817.991222:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1706 00000100:00080000:3.0:1775765817.991226:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765817.991232:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765817.991248:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00100000:3.0:1775765817.991256:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765817.991262:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741898624:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765817.991278:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741899008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765817.991345:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765817.991382:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765817/real 1775765817] req@ffff90c218da2d80 x1862023741898624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765817.991402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741898624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765817.991410:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765817.991414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765817.991417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741898624:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765817.991422:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765817.991440:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765817.991469:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765817/real 1775765817] req@ffff90c218da3800 x1862023741898752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765817.991483:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741898752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765817.991492:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765817.991496:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765817.991500:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765817.991507:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765817.991530:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765817.991540:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765817/real 1775765817] req@ffff90c218da1880 x1862023741898880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765817.991549:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741898880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765817.991559:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765817.991562:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765817.991564:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741898880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765817.991584:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899008:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765817.991597:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765817.991603:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765817/real 1775765817] req@ffff90c218da0e00 x1862023741899008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765817.991610:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741899008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765842 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765817.991620:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765817.991624:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765817.991628:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899008:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765819.015221:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765820.039160:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765821.063019:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765822.087234:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765823.110986:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765823.110994:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765823.110999:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765823.111009:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1711 00000100:00080000:3.0:1775765823.111020:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765823.111028:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765823.111066:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765823.111072:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741899136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765823.111098:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765823.111101:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765823.111102:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765823.111105:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1711 00000100:00080000:3.0:1775765823.111109:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765823.111113:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765823.111121:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765823.111125:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741899136:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765823.111135:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741899264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765823.111144:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765823.111146:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765823.111148:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765823.111151:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1711 00000100:00080000:3.0:1775765823.111154:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765823.111156:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765823.111159:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765823.111166:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741899392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765823.111174:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765823.111177:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765823.111178:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775765823.111184:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765823.111185:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1711 00000100:00080000:3.0:1775765823.111191:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765823.111196:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765823.111202:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765823.111217:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741899520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765823.111223:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765823/real 1775765823] req@ffff90c218da0380 x1862023741899136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765823.111250:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741899136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765823.111263:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765823.111266:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765823.111269:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741899136:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765823.111287:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765823.111314:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765823.111325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765823/real 1775765823] req@ffff90c218da3480 x1862023741899264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765823.111338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741899264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765823.111346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765823.111350:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765823.111353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765823.111360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899392:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765823.111380:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765823.111421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765823/real 1775765823] req@ffff90c218da0700 x1862023741899392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765823.111435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741899392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765823.111443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765823.111456:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765823.111459:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765823.111467:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899520:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765823.111484:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765823.111491:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765823/real 1775765823] req@ffff90c218da3100 x1862023741899520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765823.111499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741899520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765823.111506:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765823.111508:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765823.111510:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899520:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765824.135068:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765825.159139:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765826.183123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765827.207055:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765828.231026:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765828.231038:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765828.231043:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765828.231056:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1716 00010000:00010000:0.0:1775765828.231057:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765828.231085:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765828.231094:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765828.231127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741899648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765828.231149:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765828.231152:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765828.231153:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765828.231156:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1716 00000100:00080000:1.0:1775765828.231159:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765828.231161:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765828.231169:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765828.231177:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741899776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765828.231186:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765828.231187:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741899648:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765828.231189:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765828.231192:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765828.231196:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1716 00000100:00080000:1.0:1775765828.231198:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765828.231200:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765828.231203:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765828.231211:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741899904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765828.231221:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765828.231224:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765828.231225:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765828.231228:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1716 00000100:00080000:1.0:1775765828.231237:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765828.231240:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765828.231243:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765828.231251:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741900032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765828.231296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765828.231332:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765828.231372:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900032:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765829.255076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765830.279048:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765831.303126:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775765831.303205:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775765831.303276:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775765831.303297:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765831.303369:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765831.303381:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775765831.303394:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775765831.303425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765828/real 1775765831] req@ffff90c203495c00 x1862023741899776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765831.303462:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741899776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765831.303481:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765831.303488:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765831.303495:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765831.303507:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765828/real 1775765831] req@ffff90c203494380 x1862023741899904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765831.303526:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741899904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765831.303538:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765831.303542:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765831.303546:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741899904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765831.303552:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765828/real 1775765831] req@ffff90c203496d80 x1862023741900032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765831.303563:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741900032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765831.303572:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765831.303576:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765831.303579:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900032:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775765831.303586:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765828/real 1775765831] req@ffff90c203495880 x1862023741899648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765831.303602:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741899648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765831.303615:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765831.303619:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765831.303626:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741899648:192.168.202.155@tcp:250:kworker.0 00010000:00010000:0.0:1775765832.327014:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765833.351037:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765833.351050:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765833.351055:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765833.351065:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1721 00000100:00080000:3.0:1775765833.351075:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765833.351083:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765833.351109:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765833.351126:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741900160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765833.351160:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765833.351163:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765833.351165:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765833.351169:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1721 00000100:00080000:3.0:1775765833.351173:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765833.351177:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765833.351187:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765833.351200:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741900160:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765833.351204:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741900288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765833.351219:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765833.351223:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765833.351225:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765833.351230:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1721 00000100:00080000:3.0:1775765833.351236:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765833.351241:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765833.351248:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775765833.351262:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765833.351266:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741900416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765833.351278:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765833.351281:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765833.351282:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765833.351286:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1721 00000100:00080000:3.0:1775765833.351316:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765833.351321:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765833.351327:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765833.351328:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765833/real 1775765833] req@ffff90c218da1f80 x1862023741900160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765833.351341:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741900544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775765833.351349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741900160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765833.351363:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765833.351367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765833.351371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741900160:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765833.351392:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765833.351411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765833.351421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765833/real 1775765833] req@ffff90c218da0000 x1862023741900288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765833.351430:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741900288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765833.351443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765833.351446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765833.351450:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765833.351454:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765833.351472:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765833.351484:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765833/real 1775765833] req@ffff90c218da0a80 x1862023741900416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765833.351506:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741900416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765833.351516:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765833.351527:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765833.351532:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900416:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765833.351540:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900544:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765833.351565:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765833.351577:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765833/real 1775765833] req@ffff90c218da3100 x1862023741900544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765833.351609:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741900544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765833.351622:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765833.351626:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765833.351630:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900544:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765834.375035:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765835.399091:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765836.423071:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765837.447086:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765838.470962:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765838.470970:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765838.470975:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765838.470983:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1726 00000100:00080000:3.0:1775765838.470993:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765838.471000:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765838.471029:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765838.471034:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741900672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765838.471057:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765838.471059:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765838.471061:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765838.471064:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1726 00000100:00080000:3.0:1775765838.471066:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765838.471068:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765838.471077:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765838.471085:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741900800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765838.471094:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775765838.471098:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741900672:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765838.471098:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765838.471101:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765838.471109:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1726 00000100:00080000:3.0:1775765838.471111:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765838.471113:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765838.471117:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765838.471127:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741900928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765838.471133:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765838.471135:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765838.471137:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765838.471140:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1726 00000100:00080000:3.0:1775765838.471142:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765838.471144:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765838.471148:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765838.471167:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741901056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765838.471173:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765838.471217:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765838/real 1775765838] req@ffff90c218da0700 x1862023741900672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765838.471242:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741900672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765838.471266:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765838.471269:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765838.471272:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741900672:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765838.471304:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765838.471321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765838.471330:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765838/real 1775765838] req@ffff90c218da3480 x1862023741900800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765838.471339:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741900800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765838.471344:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765838.471347:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765838.471349:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765838.471353:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900928:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765838.471363:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765838.471376:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765838/real 1775765838] req@ffff90c218da0380 x1862023741900928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765838.471383:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741900928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765838.471388:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765838.471390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765838.471392:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741900928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765838.471396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901056:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765838.471405:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765838.471410:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765838/real 1775765838] req@ffff90c218da0e00 x1862023741901056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765838.471416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741901056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765838.471422:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765838.471424:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765838.471437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901056:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765839.495063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765840.519083:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765841.543031:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765842.567034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765843.591009:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765843.591020:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765843.591027:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765843.591038:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1732 00000100:00080000:3.0:1775765843.591052:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765843.591060:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765843.591127:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741901184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765843.591167:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765843.591176:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765843.591184:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765843.591189:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765843.591198:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1732 00000100:00080000:3.0:1775765843.591228:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765843.591244:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765843.591261:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765843.591285:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741901184:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765843.591303:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741901312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765843.591322:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765843.591326:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765843.591329:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765843.591334:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1732 00000100:00080000:3.0:1775765843.591358:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765843.591363:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765843.591370:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765843.591371:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765843.591394:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741901440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765843.591406:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765843.591411:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765843.591413:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765843.591418:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1732 00000100:00100000:2.0:1775765843.591424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765843/real 1775765843] req@ffff90c218da1880 x1862023741901184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765843.591425:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765843.591438:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765843.591449:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765843.591499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741901184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765843.591506:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741901568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765843.591573:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765843.591581:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765843.591590:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741901184:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765843.591624:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765843.591656:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765843.591671:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765843/real 1775765843] req@ffff90c218da3800 x1862023741901312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765843.591688:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741901312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765843.591698:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765843.591702:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765843.591706:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765843.591713:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765843.591731:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765843.591741:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765843/real 1775765843] req@ffff90c218da2d80 x1862023741901440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765843.591755:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741901440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765843.591764:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765843.591768:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765843.591771:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765843.591797:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901568:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765843.591815:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765843.591824:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765843/real 1775765843] req@ffff90c218da0e00 x1862023741901568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765843.591836:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741901568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765843.591846:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765843.591848:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765843.591851:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901568:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765844.615103:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765845.639146:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765846.663167:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765847.687046:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765848.711007:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765848.711018:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765848.711024:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765848.711034:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1737 00000100:00080000:3.0:1775765848.711047:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765848.711055:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765848.711119:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741901696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765848.711135:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765848.711149:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765848.711166:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765848.711168:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765848.711174:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1737 00000100:00080000:3.0:1775765848.711179:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765848.711183:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765848.711193:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765848.711205:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741901696:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765848.711208:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741901824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765848.711219:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765848.711223:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765848.711225:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765848.711229:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1737 00000100:00080000:3.0:1775765848.711233:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765848.711236:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765848.711241:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765848.711251:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741901952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765848.711260:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765848.711263:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765848.711265:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775765848.711267:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765848.711269:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1737 00000100:00080000:3.0:1775765848.711272:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765848.711276:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765848.711281:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765848.711292:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741902080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765848.711318:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765848/real 1775765848] req@ffff90c218da0380 x1862023741901696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765848.711339:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741901696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765848.711352:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765848.711355:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765848.711358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741901696:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765848.711379:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765848.711415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765848.711425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765848/real 1775765848] req@ffff90c218da3480 x1862023741901824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765848.711435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741901824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765848.711442:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765848.711444:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765848.711446:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765848.711450:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765848.711461:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765848.711480:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765848/real 1775765848] req@ffff90c218da0700 x1862023741901952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765848.711488:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741901952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765848.711493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765848.711495:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765848.711497:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741901952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765848.711501:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902080:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765848.711511:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765848.711517:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765848/real 1775765848] req@ffff90c218da3100 x1862023741902080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765873 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765848.711525:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741902080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765873 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765848.711537:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765848.711539:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765848.711542:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902080:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765849.735115:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765850.759056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765851.783157:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765852.807024:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765853.830960:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765853.830970:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765853.830975:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765853.830983:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1742 00000100:00080000:3.0:1775765853.830993:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765853.831000:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765853.831036:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741902208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765853.831041:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765853.831064:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765853.831067:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765853.831069:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765853.831072:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1742 00000100:00080000:3.0:1775765853.831075:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765853.831078:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765853.831086:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765853.831096:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741902336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765853.831120:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765853.831123:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765853.831125:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765853.831130:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741902208:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765853.831132:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1742 00000100:00080000:3.0:1775765853.831139:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765853.831143:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765853.831148:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765853.831161:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741902464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765853.831168:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765853.831171:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765853.831172:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765853.831175:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1742 00000100:00080000:3.0:1775765853.831187:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765853.831189:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765853.831193:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765853.831201:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741902592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765853.831205:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765853.831250:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765853/real 1775765853] req@ffff90c218da0a80 x1862023741902208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765853.831273:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741902208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765853.831286:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765853.831292:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765853.831296:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741902208:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765853.831316:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902336:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765853.831337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765853.831347:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765853/real 1775765853] req@ffff90c218da0000 x1862023741902336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765853.831357:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741902336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765853.831363:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765853.831366:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765853.831368:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765853.831373:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902464:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765853.831385:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765853.831392:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765853/real 1775765853] req@ffff90c218da1f80 x1862023741902464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765853.831400:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741902464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765853.831406:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765853.831408:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765853.831410:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765853.831414:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902592:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765853.831425:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765853.831430:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765853/real 1775765853] req@ffff90c218da3100 x1862023741902592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765853.831437:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741902592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765878 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765853.831444:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765853.831446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765853.831448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902592:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765854.855133:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765855.879026:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765856.903038:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765857.927020:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765858.951039:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765858.951048:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765858.951053:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765858.951076:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1747 00000100:00080000:3.0:1775765858.951089:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765858.951101:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775765858.951126:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765858.951145:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741902720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765858.951168:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765858.951171:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765858.951172:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765858.951176:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1747 00000100:00080000:3.0:1775765858.951178:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765858.951181:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765858.951190:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765858.951201:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741902848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765858.951209:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765858.951216:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741902720:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765858.951216:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765858.951229:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765858.951233:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1747 00000100:00080000:3.0:1775765858.951236:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765858.951238:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765858.951242:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765858.951253:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741902976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765858.951259:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765858.951262:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765858.951264:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765858.951267:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1747 00000100:00080000:3.0:1775765858.951269:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765858.951272:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765858.951276:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765858.951284:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741903104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765858.951305:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765858.951349:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765858/real 1775765858] req@ffff90c218da0700 x1862023741902720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765858.951367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741902720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765858.951376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765858.951380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765858.951384:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741902720:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765858.951409:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765858.951424:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765858.951433:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765858/real 1775765858] req@ffff90c218da3480 x1862023741902848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765858.951441:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741902848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765858.951447:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765858.951449:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765858.951452:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765858.951455:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765858.951465:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765858.951471:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765858/real 1775765858] req@ffff90c218da0380 x1862023741902976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765858.951478:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741902976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765858.951487:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765858.951490:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765858.951492:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741902976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765858.951496:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903104:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765858.951514:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765858.951520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765858/real 1775765858] req@ffff90c218da0e00 x1862023741903104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765858.951527:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741903104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765858.951534:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765858.951536:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765858.951538:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903104:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765859.975015:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765860.999136:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765862.023078:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765863.047125:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765864.071063:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765864.071075:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765864.071083:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765864.071098:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1752 00000100:00080000:3.0:1775765864.071114:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765864.071123:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765864.071177:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741903232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765864.071210:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765864.071211:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765864.071221:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765864.071223:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765864.071231:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1752 00000100:00080000:3.0:1775765864.071239:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765864.071243:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765864.071254:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765864.071268:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741903232:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765864.071271:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741903360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765864.071299:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765864.071303:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765864.071305:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765864.071310:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1752 00000100:00080000:3.0:1775765864.071314:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765864.071318:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765864.071323:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765864.071335:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741903488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765864.071344:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765864.071347:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765864.071349:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765864.071353:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1752 00000100:00080000:3.0:1775765864.071366:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775765864.071371:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765864.071379:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765864.071386:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765864.071403:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741903616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765864.071445:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765864/real 1775765864] req@ffff90c218da2d80 x1862023741903232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765864.071476:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741903232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765864.071493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765864.071499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765864.071505:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741903232:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765864.071536:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765864.071562:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765864.071575:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765864/real 1775765864] req@ffff90c218da3800 x1862023741903360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765864.071587:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741903360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765864.071595:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765864.071599:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765864.071602:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765864.071607:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903488:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765864.071645:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765864.071656:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765864/real 1775765864] req@ffff90c218da1880 x1862023741903488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765864.071669:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741903488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765864.071695:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765864.071699:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765864.071702:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765864.071708:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903616:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765864.071726:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765864.071736:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765864/real 1775765864] req@ffff90c218da0e00 x1862023741903616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765864.071748:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741903616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765864.071759:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765864.071762:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765864.071765:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903616:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765865.095050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765866.119055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765867.143143:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765868.167116:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765869.191005:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765869.191014:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765869.191019:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765869.191027:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1757 00000100:00080000:1.0:1775765869.191039:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765869.191045:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765869.191082:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741903744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775765869.191084:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765869.191106:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765869.191108:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765869.191110:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765869.191113:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1757 00000100:00080000:1.0:1775765869.191115:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765869.191118:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765869.191126:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765869.191136:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741903872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765869.191141:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741903744:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765869.191150:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765869.191153:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765869.191154:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765869.191157:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1757 00000100:00080000:1.0:1775765869.191159:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765869.191161:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765869.191165:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765869.191172:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741904000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765869.191178:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765869.191180:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765869.191181:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765869.191183:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1757 00000100:00080000:1.0:1775765869.191186:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765869.191189:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765869.191192:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765869.191199:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741904128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765869.191221:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765869.191256:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765869/real 1775765869] req@ffff90c203497100 x1862023741903744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765869.191272:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741903744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765869.191280:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765869.191289:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765869.191293:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741903744:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765869.191314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903872:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765869.191332:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765869.191341:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765869/real 1775765869] req@ffff90c203495f80 x1862023741903872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765869.191350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741903872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765869.191369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765869.191372:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765869.191374:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741903872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765869.191378:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765869.191394:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765869.191402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765869/real 1775765869] req@ffff90c203494000 x1862023741904000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765869.191410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741904000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765869.191415:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765869.191417:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765869.191419:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765869.191423:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904128:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765869.191437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765869.191443:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765869/real 1775765869] req@ffff90c203497480 x1862023741904128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765869.191450:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741904128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765869.191458:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765869.191461:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765869.191464:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904128:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765870.215114:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765871.239046:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765872.263076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765873.287030:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765874.311017:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765874.311029:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765874.311036:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765874.311050:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1762 00000100:00080000:0.0:1775765874.311065:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765874.311077:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765874.311099:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765874.311128:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741904256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765874.311155:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765874.311159:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765874.311160:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765874.311164:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1762 00000100:00080000:0.0:1775765874.311166:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765874.311170:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765874.311181:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765874.311192:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741904384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765874.311203:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765874.311209:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765874.311211:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765874.311213:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741904256:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765874.311216:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1762 00000100:00080000:0.0:1775765874.311222:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765874.311231:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765874.311235:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765874.311246:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741904512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765874.311253:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765874.311256:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765874.311257:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765874.311261:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1762 00000100:00080000:0.0:1775765874.311264:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765874.311266:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765874.311271:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765874.311280:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741904640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765874.311303:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765874.311344:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765874/real 1775765874] req@ffff90c20363aa00 x1862023741904256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765874.311363:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741904256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765874.311375:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765874.311380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765874.311384:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741904256:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765874.311410:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765874.311432:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765874.311444:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765874/real 1775765874] req@ffff90c203638000 x1862023741904384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765874.311458:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741904384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765874.311467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765874.311471:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765874.311474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765874.311481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765874.311500:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765874.311510:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765874/real 1775765874] req@ffff90c203639180 x1862023741904512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765874.311523:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741904512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765874.311532:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765874.311535:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765874.311539:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765874.311545:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904640:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765874.311563:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765874.311573:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765874/real 1775765874] req@ffff90c203639500 x1862023741904640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765874.311585:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741904640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765874.311595:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765874.311599:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765874.311602:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904640:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765875.335049:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765876.358987:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765877.383003:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765878.407173:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765879.430967:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765879.430975:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765879.430979:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765879.430988:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1767 00000100:00080000:0.0:1775765879.430998:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765879.431004:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765879.431048:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765879.431050:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741904768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765879.431073:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765879.431075:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765879.431077:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765879.431079:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1767 00000100:00080000:0.0:1775765879.431082:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765879.431085:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765879.431093:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765879.431106:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741904896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765879.431106:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741904768:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765879.431120:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765879.431123:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765879.431124:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765879.431127:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1767 00000100:00080000:0.0:1775765879.431130:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765879.431132:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765879.431135:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765879.431145:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741905024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765879.431151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765879.431154:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765879.431155:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765879.431157:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1767 00000100:00080000:0.0:1775765879.431160:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765879.431162:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765879.431166:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:0.0:1775765879.431167:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765879.431176:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741905152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765879.431203:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765879/real 1775765879] req@ffff90c203638700 x1862023741904768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765879.431216:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741904768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765879.431224:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765879.431227:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765879.431230:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741904768:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765879.431246:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904896:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765879.431259:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765879.431266:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765879/real 1775765879] req@ffff90c203639500 x1862023741904896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765879.431274:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741904896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765879.431279:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765879.431281:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765879.431283:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741904896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765879.431287:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905024:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765879.431296:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765879.431301:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765879/real 1775765879] req@ffff90c203639180 x1862023741905024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765879.431308:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741905024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765879.431323:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765879.431326:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765879.431328:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765879.431332:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905152:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765879.431343:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765879.431349:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765879/real 1775765879] req@ffff90c203638000 x1862023741905152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765879.431356:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741905152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765879.431362:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765879.431364:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765879.431366:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905152:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765880.455081:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765881.479086:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765882.503112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765883.527094:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765884.550992:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765884.551003:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765884.551009:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765884.551021:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1773 00010000:00010000:0.0:1775765884.551030:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765884.551042:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765884.551049:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765884.551084:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741905280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765884.551105:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765884.551108:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765884.551109:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765884.551112:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1773 00000100:00080000:3.0:1775765884.551117:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765884.551121:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765884.551131:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765884.551135:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741905280:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765884.551148:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741905408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765884.551161:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765884.551166:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765884.551168:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765884.551172:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1773 00000100:00080000:3.0:1775765884.551180:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765884.551185:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775765884.551189:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775765884.551193:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765884.551209:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741905536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765884.551221:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765884.551224:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765884/real 1775765884] req@ffff90c218da0380 x1862023741905280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765884.551229:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765884.551233:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765884.551242:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1773 00000100:00100000:2.0:1775765884.551246:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741905280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765884.551246:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765884.551255:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775765884.551260:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775765884.551264:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775765884.551266:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765884.551271:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741905280:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765884.551284:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741905664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765884.551301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765884.551317:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765884.551325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765884/real 1775765884] req@ffff90c218da3480 x1862023741905408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765884.551333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741905408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765884.551339:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765884.551341:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765884.551344:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765884.551347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905536:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765884.551356:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765884.551361:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765884/real 1775765884] req@ffff90c218da0700 x1862023741905536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765884.551368:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741905536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765884.551373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765884.551375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765884.551377:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765884.551380:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905664:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765884.551389:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765884.551393:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765884/real 1775765884] req@ffff90c218da3100 x1862023741905664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765884.551400:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741905664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765884.551411:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765884.551413:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765884.551415:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905664:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765885.575061:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765886.599049:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765887.623110:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765888.647042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765889.671005:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765889.671016:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765889.671021:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765889.671032:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1778 00000100:00080000:1.0:1775765889.671044:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765889.671056:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765889.671058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765889.671094:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741905792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765889.671121:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765889.671124:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765889.671126:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765889.671129:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1778 00000100:00080000:1.0:1775765889.671131:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765889.671134:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765889.671142:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765889.671153:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741905920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765889.671162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775765889.671163:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741905792:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765889.671168:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765889.671171:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765889.671175:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1778 00000100:00080000:1.0:1775765889.671177:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765889.671179:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765889.671182:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765889.671193:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741906048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765889.671204:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765889.671207:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765889.671209:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765889.671213:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1778 00000100:00080000:1.0:1775765889.671217:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765889.671221:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765889.671227:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765889.671233:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765889.671242:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741906176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765889.671269:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765889/real 1775765889] req@ffff90c203494e00 x1862023741905792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765889.671284:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741905792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765889.671291:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765889.671294:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765889.671297:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741905792:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765889.671314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765889.671330:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765889.671337:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765889/real 1775765889] req@ffff90c203497480 x1862023741905920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765889.671345:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741905920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765889.671351:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765889.671353:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765889.671355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741905920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765889.671360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906048:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765889.671370:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765889.671375:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765889/real 1775765889] req@ffff90c203494000 x1862023741906048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765889.671382:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741906048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765889.671387:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765889.671389:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765889.671390:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765889.671394:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906176:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765889.671403:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765889.671416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765889/real 1775765889] req@ffff90c203495f80 x1862023741906176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765914 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765889.671424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741906176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765914 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765889.671430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765889.671432:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765889.671434:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906176:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765890.695049:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765891.719078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765892.743035:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765893.767095:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765894.790952:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765894.790961:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765894.790965:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765894.790973:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1783 00000100:00080000:1.0:1775765894.790982:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765894.790988:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765894.791021:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741906304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775765894.791025:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765894.791044:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765894.791047:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765894.791048:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765894.791051:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1783 00000100:00080000:1.0:1775765894.791053:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765894.791055:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765894.791063:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765894.791072:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741906432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765894.791082:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765894.791086:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765894.791088:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765894.791094:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741906304:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765894.791095:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1783 00000100:00080000:1.0:1775765894.791102:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765894.791107:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765894.791113:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765894.791126:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741906560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765894.791138:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765894.791141:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765894.791143:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765894.791148:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1783 00000100:00080000:1.0:1775765894.791153:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765894.791157:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765894.791163:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765894.791177:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741906688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765894.791185:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765894.791219:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765894.791235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906688:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765895.815002:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765896.839010:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765897.863086:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775765897.863160:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775765897.863200:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775765897.863218:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775765897.863257:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775765897.863267:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775765897.863278:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775765897.863305:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765894/real 1775765897] req@ffff90c203496d80 x1862023741906560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765897.863334:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741906560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765897.863347:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765897.863353:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765897.863357:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765897.863366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765894/real 1775765897] req@ffff90c203494380 x1862023741906688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765897.863380:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741906688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765897.863391:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765897.863394:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765897.863398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906688:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775765897.863405:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765894/real 1775765897] req@ffff90c203497100 x1862023741906304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765897.863419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741906304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765897.863433:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765897.863436:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765897.863440:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741906304:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765897.863447:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765894/real 1775765897] req@ffff90c203495880 x1862023741906432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765919 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765897.863468:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741906432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765919 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765897.863476:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765897.863479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765897.863483:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906432:192.168.202.155@tcp:38:kworker.0 00010000:00010000:3.0:1775765898.887019:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765899.910993:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765899.911001:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765899.911005:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765899.911016:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1788 00000100:00080000:1.0:1775765899.911026:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765899.911032:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765899.911063:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741906816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765899.911096:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:0.0:1775765899.911099:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765899.911099:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765899.911103:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765899.911107:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1788 00000100:00080000:1.0:1775765899.911111:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765899.911117:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765899.911127:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765899.911146:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741906944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765899.911147:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741906816:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765899.911170:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765899.911174:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765899.911176:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765899.911183:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1788 00000100:00080000:1.0:1775765899.911188:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765899.911193:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765899.911199:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765899.911216:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741907072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765899.911228:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765899.911233:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765899.911235:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765899.911239:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1788 00000800:00000100:2.0:1775765899.911246:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775765899.911248:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765899.911256:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765899.911263:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765899.911289:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741907200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765899.911306:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765899/real 1775765899] req@ffff90c203495c00 x1862023741906816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765899.911334:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741906816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765899.911347:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765899.911353:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765899.911358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741906816:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765899.911399:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765899.911434:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765899.911450:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765899/real 1775765899] req@ffff90c203497b80 x1862023741906944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765899.911463:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741906944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765899.911472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765899.911476:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765899.911479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741906944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765899.911485:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765899.911507:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765899.911519:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765899/real 1775765899] req@ffff90c203496300 x1862023741907072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765899.911530:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741907072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765899.911539:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765899.911543:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765899.911546:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765899.911553:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907200:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765899.911578:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765899.911588:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765899/real 1775765899] req@ffff90c203496a00 x1862023741907200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765899.911607:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741907200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765924 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765899.911618:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765899.911621:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765899.911625:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907200:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765900.935060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765901.959112:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765902.983066:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765904.007054:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765905.030952:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765905.030960:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765905.030964:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765905.030977:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1793 00000100:00080000:1.0:1775765905.030987:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765905.030993:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765905.031024:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741907328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775765905.031026:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765905.031047:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765905.031050:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765905.031051:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765905.031054:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1793 00000100:00080000:1.0:1775765905.031057:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765905.031059:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765905.031067:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765905.031079:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741907456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765905.031080:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741907328:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775765905.031095:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765905.031098:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765905.031099:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765905.031102:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1793 00000100:00080000:1.0:1775765905.031104:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765905.031107:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765905.031110:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765905.031118:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741907584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765905.031125:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765905.031127:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765905.031128:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765905.031131:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1793 00000100:00080000:1.0:1775765905.031134:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765905.031136:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765905.031140:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765905.031144:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775765905.031151:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741907712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765905.031176:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765905/real 1775765905] req@ffff90c203496680 x1862023741907328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765905.031190:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741907328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765905.031198:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765905.031201:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765905.031204:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741907328:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765905.031220:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765905.031234:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765905.031241:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765905/real 1775765905] req@ffff90c203494a80 x1862023741907456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765905.031249:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741907456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765905.031259:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765905.031261:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765905.031263:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765905.031266:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907584:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765905.031275:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765905.031280:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765905/real 1775765905] req@ffff90c203495180 x1862023741907584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765905.031287:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741907584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765905.031292:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765905.031294:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765905.031295:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765905.031299:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907712:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765905.031308:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765905.031313:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765905/real 1775765905] req@ffff90c203495500 x1862023741907712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765905.031319:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741907712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765905.031325:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765905.031327:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765905.031329:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907712:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765906.055099:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765907.079042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765908.103017:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765909.127114:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765910.151017:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765910.151029:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765910.151035:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765910.151057:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1798 00000100:00080000:0.0:1775765910.151071:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765910.151080:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765910.151087:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765910.151127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741907840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765910.151159:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765910.151163:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765910.151164:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765910.151168:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1798 00000100:00080000:0.0:1775765910.151171:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765910.151174:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765910.151185:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765910.151197:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741907968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765910.151201:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741907840:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765910.151228:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765910.151231:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765910.151232:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765910.151235:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1798 00000100:00080000:0.0:1775765910.151238:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765910.151241:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765910.151245:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765910.151265:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741908096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765910.151273:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765910.151275:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765910.151277:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765910.151281:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1798 00000100:00080000:0.0:1775765910.151284:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765910.151287:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765910.151292:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765910.151299:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765910.151310:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741908224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765910.151344:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765910/real 1775765910] req@ffff90c20363aa00 x1862023741907840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765910.151362:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741907840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765910.151371:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765910.151375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765910.151378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741907840:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765910.151413:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765910.151430:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765910.151439:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765910/real 1775765910] req@ffff90c20363b100 x1862023741907968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765910.151447:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741907968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765910.151453:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765910.151456:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765910.151458:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741907968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765910.151462:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765910.151473:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765910.151479:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765910/real 1775765910] req@ffff90c20363bb80 x1862023741908096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765910.151486:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741908096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765910.151491:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765910.151493:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765910.151495:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765910.151499:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908224:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765910.151509:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765910.151514:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765910/real 1775765910] req@ffff90c20363a680 x1862023741908224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765910.151522:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741908224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765910.151528:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765910.151530:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765910.151532:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908224:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765911.175087:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765912.199090:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765913.223063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765914.247025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765915.271056:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765915.271068:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765915.271074:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765915.271085:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1803 00010000:00010000:0.0:1775765915.271089:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765915.271100:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765915.272273:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765915.272321:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741908352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765915.272348:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765915.272351:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765915.272353:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765915.272357:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1803 00000100:00080000:3.0:1775765915.272360:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765915.272363:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765915.272374:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765915.272384:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741908480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765915.272402:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741908352:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765915.272403:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765915.272411:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765915.272420:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765915.272427:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1803 00000100:00080000:3.0:1775765915.272430:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765915.272435:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765915.272442:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765915.272469:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741908608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765915.272481:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765915.272486:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765915.272488:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765915.272491:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1803 00000100:00080000:3.0:1775765915.272527:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765915.272532:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765915.272540:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765915.272549:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765915.272572:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741908736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765915.272629:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765915/real 1775765915] req@ffff90c218da1f80 x1862023741908352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765915.272647:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741908352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765915.272657:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765915.272661:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765915.272664:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741908352:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765915.272686:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765915.272707:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765915.272716:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765915/real 1775765915] req@ffff90c218da0000 x1862023741908480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765915.272726:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741908480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765915.272732:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765915.272735:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765915.272737:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765915.272741:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765915.272753:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765915.272759:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765915/real 1775765915] req@ffff90c218da0a80 x1862023741908608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765915.272767:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741908608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765915.272773:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765915.272775:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765915.272777:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765915.272782:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908736:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765915.272793:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765915.272799:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765915/real 1775765915] req@ffff90c218da3100 x1862023741908736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765915.272807:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741908736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765915.272814:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765915.272816:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765915.272818:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908736:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765916.295090:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765917.319179:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765918.343184:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765919.367076:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765920.390993:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765920.391002:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765920.391006:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765920.391017:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1808 00000100:00080000:1.0:1775765920.391027:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775765920.391029:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765920.391035:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765920.391071:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741908864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765920.391097:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765920.391102:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765920.391104:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765920.391109:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1808 00000100:00080000:1.0:1775765920.391114:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765920.391118:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765920.391130:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765920.391133:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741908864:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765920.391150:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741908992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765920.391165:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765920.391169:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765920.391170:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765920.391174:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1808 00000100:00080000:1.0:1775765920.391178:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765920.391181:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765920.391186:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765920.391196:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741909120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765920.391204:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765920.391210:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1775765920.391210:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775765920.391213:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765920.391218:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1808 00000100:00080000:1.0:1775765920.391222:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765920.391224:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765920.391228:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765920.391236:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741909248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765920.391267:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765920/real 1775765920] req@ffff90c203494700 x1862023741908864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765920.391283:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741908864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765920.391292:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765920.391296:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765920.391299:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741908864:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765920.391319:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908992:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765920.391337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765920.391345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765920/real 1775765920] req@ffff90c203495500 x1862023741908992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765920.391355:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741908992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765920.391361:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765920.391364:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765920.391366:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741908992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765920.391370:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765920.391381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765920.391387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765920/real 1775765920] req@ffff90c203495180 x1862023741909120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765920.391395:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741909120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765920.391401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765920.391402:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765920.391404:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765920.391408:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909248:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765920.391418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765920.391424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765920/real 1775765920] req@ffff90c203494a80 x1862023741909248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765920.391431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741909248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765920.391443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765920.391446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765920.391448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909248:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765921.415085:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765922.439155:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765923.463080:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765924.487072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765925.511057:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765925.511074:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765925.511083:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765925.511102:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1813 00010000:00010000:1.0:1775765925.511104:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765925.511125:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765925.511138:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765925.511202:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741909376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765925.511248:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765925.511254:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765925.511257:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765925.511264:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1813 00000100:00080000:3.0:1775765925.511271:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765925.511278:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765925.511296:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765925.511321:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741909504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765925.511323:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741909376:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765925.511351:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765925.511358:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765925.511361:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765925.511369:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1813 00000100:00080000:3.0:1775765925.511377:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765925.511386:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765925.511396:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765925.511442:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741909632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765925.511450:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775765925.511473:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765925.511480:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765925.511485:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765925.511495:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1813 00000100:00080000:3.0:1775765925.511505:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775765925.511516:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765925/real 1775765925] req@ffff90c218da0700 x1862023741909376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775765925.511517:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765925.511537:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765925.511551:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741909376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775765925.511568:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741909760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765925.511578:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765925.511591:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765925.511600:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741909376:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765925.511638:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765925.511668:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765925.511683:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765925/real 1775765925] req@ffff90c218da3480 x1862023741909504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765925.511697:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741909504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765925.511707:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765925.511712:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765925.511715:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765925.511723:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765925.511741:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765925.511751:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765925/real 1775765925] req@ffff90c218da0380 x1862023741909632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765925.511764:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741909632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765925.511773:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765925.511776:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765925.511779:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765925.511785:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909760:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765925.511802:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765925.511811:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765925/real 1775765925] req@ffff90c218da0e00 x1862023741909760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765925.511823:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741909760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765925.511834:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765925.511837:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765925.511841:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741909760:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765926.535217:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765927.559049:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765928.583103:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765929.607096:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765930.631109:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765930.631136:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765930.631141:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765930.631162:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1819 00000100:00080000:3.0:1775765930.631175:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775765930.631177:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765930.631184:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765930.631223:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741909888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765930.631248:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765930.631251:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765930.631254:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765930.631262:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1819 00000100:00080000:3.0:1775765930.631266:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765930.631270:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765930.631279:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765930.631295:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741909888:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775765930.631295:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741910016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765930.631322:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765930.631324:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765930.631326:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765930.631329:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1819 00000100:00080000:3.0:1775765930.631333:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765930.631336:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765930.631339:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765930.631348:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741910144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765930.631362:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765930.631365:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765930.631366:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765930.631390:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1819 00000100:00080000:3.0:1775765930.631402:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765930.631405:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765930.631410:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765930.631418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765930.631428:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741910272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765930.631478:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765930/real 1775765930] req@ffff90c218da1880 x1862023741909888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765930.631501:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741909888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765930.631513:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765930.631518:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765930.631523:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741909888:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765930.631554:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765930.631579:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765930.631590:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765930/real 1775765930] req@ffff90c218da3800 x1862023741910016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765930.631599:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741910016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765930.631605:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765930.631608:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765930.631610:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765930.631614:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765930.631626:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765930.631632:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765930/real 1775765930] req@ffff90c218da2d80 x1862023741910144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765930.631639:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741910144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765930.631649:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765930.631651:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765930.631653:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765930.631657:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910272:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765930.631667:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765930.631672:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765930/real 1775765930] req@ffff90c218da0e00 x1862023741910272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765955 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765930.631679:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741910272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765955 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765930.631687:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765930.631688:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765930.631690:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910272:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765931.655019:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765932.679149:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765933.703018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765934.727090:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765935.751037:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765935.751067:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765935.751073:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765935.751085:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1824 00000100:00080000:1.0:1775765935.751097:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765935.751107:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765935.751136:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765935.751171:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741910400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765935.751203:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765935.751213:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765935.751215:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765935.751218:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1824 00000100:00080000:1.0:1775765935.751222:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765935.751226:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765935.751236:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765935.751250:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741910528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765935.751264:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765935.751283:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765935.751285:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765935.751291:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1824 00000100:00080000:1.0:1775765935.751294:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765935.751304:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765935.751311:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741910400:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775765935.751316:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765935.751358:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741910656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765935.751367:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765935.751370:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765935.751371:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765935.751374:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1824 00000100:00080000:1.0:1775765935.751377:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765935.751380:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765935.751384:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765935.751396:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741910784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765935.751477:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765935.751547:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765935/real 1775765935] req@ffff90c203496680 x1862023741910400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765935.751573:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741910400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765935.751590:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765935.751596:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765935.751602:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741910400:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765935.751636:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910528:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765935.751672:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765935.751686:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765935/real 1775765935] req@ffff90c203496a00 x1862023741910528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765935.751701:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741910528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765935.751711:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765935.751715:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765935.751719:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765935.751726:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765935.751745:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765935.751755:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765935/real 1775765935] req@ffff90c203496300 x1862023741910656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765935.751774:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741910656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765935.751791:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765935.751797:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765935.751804:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765935.751815:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910784:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765935.751854:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765935.751913:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765935/real 1775765935] req@ffff90c203497b80 x1862023741910784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765960 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765935.751932:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741910784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765960 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765935.751946:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765935.751950:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765935.751954:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741910784:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765936.775094:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765937.799020:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765938.823131:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765939.847093:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765940.871029:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:1.0:1775765940.871030:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765940.871042:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765940.871047:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765940.871058:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1829 00000100:00080000:0.0:1775765940.871071:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765940.871077:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765940.871124:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741910912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765940.871149:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765940.871154:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765940.871156:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765940.871159:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1829 00000100:00080000:0.0:1775765940.871161:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765940.871164:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765940.871173:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765940.871183:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741911040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765940.871191:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765940.871196:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765940.871197:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765940.871197:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741910912:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765940.871204:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1829 00000100:00080000:0.0:1775765940.871207:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765940.871209:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765940.871212:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765940.871220:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741911168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765940.871227:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765940.871230:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765940.871231:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765940.871233:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1829 00000100:00080000:0.0:1775765940.871236:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765940.871238:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765940.871242:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765940.871248:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741911296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765940.871268:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765940.871331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765940/real 1775765940] req@ffff90c203638380 x1862023741910912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765940.871348:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741910912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765940.871358:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765940.871361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765940.871364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741910912:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765940.871386:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765940.871403:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765940.871411:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765940/real 1775765940] req@ffff90c203639f80 x1862023741911040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765940.871419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741911040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765940.871429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765940.871431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765940.871434:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765940.871437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765940.871447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765940.871453:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765940/real 1775765940] req@ffff90c203638e00 x1862023741911168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765940.871460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741911168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765940.871465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765940.871467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765940.871468:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765940.871472:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911296:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765940.871482:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765940.871487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765940/real 1775765940] req@ffff90c20363b480 x1862023741911296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765940.871494:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741911296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765965 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765940.871501:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765940.871503:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765940.871505:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911296:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765941.895125:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765942.919028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765943.943145:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765944.967070:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765945.991023:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765945.991035:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765945.991040:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765945.991052:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1834 00000100:00080000:0.0:1775765945.991064:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765945.991074:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765945.991160:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741911424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765945.991160:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765945.991197:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765945.991201:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765945.991203:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765945.991211:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1834 00000100:00080000:0.0:1775765945.991216:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765945.991220:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765945.991246:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765945.991265:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741911552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765945.991277:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765945.991281:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765945.991283:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765945.991288:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1834 00000100:00080000:0.0:1775765945.991292:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765945.991299:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765945.991301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741911424:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775765945.991305:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765945.991331:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741911680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765945.991343:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765945.991347:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765945.991349:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765945.991353:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1834 00000100:00080000:0.0:1775765945.991357:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765945.991360:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765945.991366:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765945.991376:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741911808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765945.991400:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765945.991467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765945/real 1775765945] req@ffff90c20363b800 x1862023741911424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765945.991485:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741911424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765945.991494:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765945.991498:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765945.991501:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741911424:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765945.991508:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911552:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765945.991527:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765945.991536:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765945/real 1775765945] req@ffff90c20363b480 x1862023741911552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765945.991546:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741911552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765945.991556:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765945.991560:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765945.991564:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765945.991590:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765945.991615:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765945.991628:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765945/real 1775765945] req@ffff90c203638e00 x1862023741911680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765945.991637:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741911680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765945.991644:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765945.991647:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765945.991651:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765945.991658:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911808:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765945.991681:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765945.991694:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765945/real 1775765945] req@ffff90c203639f80 x1862023741911808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765945.991709:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741911808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765970 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765945.991720:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765945.991722:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765945.991725:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741911808:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765947.015147:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765948.039091:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765949.063140:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765950.087081:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765951.110970:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765951.110978:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765951.110983:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765951.110991:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1839 00000100:00080000:1.0:1775765951.111003:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765951.111010:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765951.111043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765951.111050:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741911936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765951.111074:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765951.111077:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765951.111078:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765951.111081:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1839 00000100:00080000:1.0:1775765951.111084:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765951.111086:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765951.111094:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765951.111104:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741912064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765951.111112:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765951.111116:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765951.111117:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765951.111119:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1839 00000100:00080000:1.0:1775765951.111121:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765951.111128:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765951.111128:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741911936:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775765951.111142:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765951.111150:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741912192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765951.111157:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765951.111159:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765951.111160:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765951.111164:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1839 00000100:00080000:1.0:1775765951.111167:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765951.111169:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765951.111172:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765951.111198:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741912320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765951.111234:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765951.111278:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765951/real 1775765951] req@ffff90c203495c00 x1862023741911936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765951.111301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741911936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765951.111321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765951.111326:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765951.111332:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741911936:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765951.111364:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765951.111390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765951.111404:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765951/real 1775765951] req@ffff90c203495880 x1862023741912064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765951.111419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741912064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765951.111428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765951.111431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765951.111434:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765951.111441:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765951.111462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765951.111473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765951/real 1775765951] req@ffff90c203497100 x1862023741912192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765951.111485:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741912192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765951.111492:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765951.111495:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765951.111498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765951.111505:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912320:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765951.111527:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765951.111538:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765951/real 1775765951] req@ffff90c203494380 x1862023741912320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765951.111551:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741912320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765951.111562:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765951.111565:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765951.111567:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912320:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765952.135051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765953.159013:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765954.183040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765955.206987:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765956.231012:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765956.231022:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765956.231027:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765956.231037:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1844 00000100:00080000:3.0:1775765956.231051:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765956.231059:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775765956.231073:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775765956.231122:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741912448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765956.231213:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765956.231216:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765956.231218:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765956.231221:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1844 00000100:00080000:3.0:1775765956.231224:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765956.231227:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765956.231238:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765956.231250:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741912576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765956.231258:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765956.231263:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765956.231264:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775765956.231265:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741912448:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765956.231269:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1844 00000100:00080000:3.0:1775765956.231275:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765956.231278:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765956.231281:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765956.231289:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741912704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765956.231303:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765956.231306:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765956.231307:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765956.231310:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1844 00000100:00080000:3.0:1775765956.231328:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765956.231330:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765956.231334:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775765956.231339:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775765956.231346:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741912832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765956.231381:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765956/real 1775765956] req@ffff90c218da0380 x1862023741912448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765956.231402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741912448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765956.231414:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765956.231419:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765956.231424:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741912448:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765956.231448:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765956.231467:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765956.231476:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765956/real 1775765956] req@ffff90c218da3480 x1862023741912576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765956.231485:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741912576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765956.231491:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765956.231493:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765956.231496:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765956.231502:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765956.231521:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765956.231532:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765956/real 1775765956] req@ffff90c218da0700 x1862023741912704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765956.231545:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741912704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765956.231554:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765956.231559:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765956.231562:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765956.231567:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912832:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765956.231592:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765956.231600:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765956/real 1775765956] req@ffff90c218da3100 x1862023741912832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765956.231608:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741912832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765956.231619:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765956.231621:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765956.231623:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741912832:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765957.255142:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765958.279055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765959.303112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765960.327084:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765961.351020:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765961.351029:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775765961.351034:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765961.351046:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1849 00000100:00080000:3.0:1775765961.351059:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765961.351067:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775765961.351133:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741912960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775765961.351139:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775765961.351167:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765961.351170:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775765961.351172:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765961.351176:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1849 00000100:00080000:3.0:1775765961.351181:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765961.351186:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765961.351198:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765961.351213:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741913088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765961.351214:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741912960:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775765961.351238:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765961.351242:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775765961.351244:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765961.351249:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1849 00000100:00080000:3.0:1775765961.351253:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765961.351257:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765961.351262:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775765961.351277:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741913216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775765961.351287:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775765961.351290:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775765961.351292:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775765961.351295:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1849 00000100:00080000:3.0:1775765961.351307:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775765961.351310:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775765961.351315:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775765961.351327:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741913344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765961.351369:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765961.351399:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765961.351435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913344:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775765962.375024:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765963.399097:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765964.423013:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775765964.423136:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775765964.423170:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775765964.423185:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775765964.423214:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775765964.423221:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775765964.423231:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775765964.423250:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765961/real 1775765964] req@ffff90c218da0000 x1862023741913088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765964.423271:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741913088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765964.423281:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765964.423285:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765964.423288:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765964.423293:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765961/real 1775765964] req@ffff90c218da1f80 x1862023741913216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765964.423301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741913216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765964.423306:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765964.423309:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765964.423311:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765964.423315:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765961/real 1775765964] req@ffff90c218da3100 x1862023741913344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765964.423322:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741913344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765964.423328:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765964.423330:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765964.423332:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913344:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775765964.423336:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765961/real 1775765964] req@ffff90c218da0a80 x1862023741912960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765964.423343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741912960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765964.423347:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765964.423349:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765964.423351:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741912960:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775765965.447050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765966.471044:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765966.471088:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765966.471114:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765966.471121:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765966.471148:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1854 00000100:00080000:0.0:1775765966.471164:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765966.471174:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775765966.471241:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741913472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765966.471269:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765966.471273:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765966.471284:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765966.471287:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1854 00000100:00080000:0.0:1775765966.471290:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765966.471293:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765966.471323:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765966.471344:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741913600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765966.471344:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741913472:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765966.471377:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765966.471399:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765966.471401:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765966.471408:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1854 00000100:00080000:0.0:1775765966.471415:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765966.471421:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765966.471427:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765966.471441:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741913728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765966.471462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765966.471481:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765966.471486:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765966.471488:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765966.471496:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1854 00000100:00080000:0.0:1775765966.471502:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765966.471507:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765966.471516:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765966.471525:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765966/real 1775765966] req@ffff90c203638380 x1862023741913472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765966.471537:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741913856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765966.471576:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741913472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765966.471592:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765966.471600:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765966.471605:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741913472:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765966.471644:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765966.471672:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765966.471692:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765966/real 1775765966] req@ffff90c20363a680 x1862023741913600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765966.471706:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741913600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765966.471717:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765966.471721:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765966.471724:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765966.471730:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765966.471753:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765966.471765:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765966/real 1775765966] req@ffff90c20363bb80 x1862023741913728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765966.471777:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741913728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765966.471784:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765966.471788:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765966.471791:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765966.471798:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913856:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765966.471820:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765966.471836:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765966/real 1775765966] req@ffff90c20363b100 x1862023741913856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765966.471849:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741913856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765966.471858:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765966.471862:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765966.471865:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741913856:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775765967.495113:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775765968.519038:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765969.543096:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765970.567020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765971.591009:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765971.591020:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765971.591025:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775765971.591033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765971.591037:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1860 00000100:00080000:1.0:1775765971.591050:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765971.591058:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775765971.591096:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741913984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765971.591126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765971.591130:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765971.591132:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765971.591136:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1860 00000100:00080000:1.0:1775765971.591140:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765971.591144:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765971.591154:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765971.591161:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741913984:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765971.591167:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741914112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765971.591179:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765971.591184:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765971.591185:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765971.591189:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1860 00000100:00080000:1.0:1775765971.591193:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765971.591197:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765971.591202:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765971.591211:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741914240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765971.591221:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775765971.591225:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775765971.591228:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765971.591231:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765971.591237:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1860 00000100:00080000:1.0:1775765971.591241:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765971.591245:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765971.591250:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775765971.591261:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741914368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765971.591264:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765971/real 1775765971] req@ffff90c203496d80 x1862023741913984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765971.591295:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741913984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775765996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765971.591304:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765971.591310:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765971.591314:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741913984:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765971.591338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765971.591362:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765971.591374:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765971/real 1775765971] req@ffff90c203495f80 x1862023741914112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765971.591385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741914112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765971.591391:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765971.591393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765971.591395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765971.591400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914240:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765971.591413:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765971.591420:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765971/real 1775765971] req@ffff90c203494000 x1862023741914240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765971.591427:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741914240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775765996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765971.591433:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765971.591435:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765971.591437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765971.591440:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914368:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765971.591450:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765971.591455:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765971/real 1775765971] req@ffff90c203497480 x1862023741914368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765971.591462:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741914368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775765996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765971.591473:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765971.591475:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765971.591477:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914368:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765972.615016:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775765973.639052:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765974.663046:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765975.687164:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765976.711063:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765976.711072:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765976.711077:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765976.711086:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1865 00000100:00080000:0.0:1775765976.711098:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765976.711106:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765976.711128:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765976.711145:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741914496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765976.711167:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765976.711169:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765976.711171:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765976.711174:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1865 00000100:00080000:0.0:1775765976.711177:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765976.711180:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765976.711191:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765976.711205:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741914624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765976.711216:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741914496:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765976.711238:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765976.711243:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765976.711245:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765976.711250:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1865 00000100:00080000:0.0:1775765976.711254:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765976.711257:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765976.711262:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765976.711286:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741914752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765976.711292:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765976.711295:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765976.711296:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765976.711298:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1865 00000100:00080000:0.0:1775765976.711301:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765976.711304:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765976.711307:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765976.711318:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741914880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765976.711327:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765976.711379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765976/real 1775765976] req@ffff90c20363aa00 x1862023741914496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765976.711404:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741914496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765976.711419:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765976.711424:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765976.711444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741914496:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765976.711475:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765976.711496:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765976.711505:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765976/real 1775765976] req@ffff90c203638000 x1862023741914624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765976.711514:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741914624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765976.711521:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765976.711524:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765976.711526:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765976.711531:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765976.711542:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765976.711547:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765976/real 1775765976] req@ffff90c203639180 x1862023741914752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765976.711555:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741914752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765976.711560:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765976.711561:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765976.711563:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765976.711567:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914880:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765976.711584:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765976.711595:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765976/real 1775765976] req@ffff90c203639500 x1862023741914880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766001 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765976.711607:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741914880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766001 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765976.711618:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765976.711622:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765976.711625:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741914880:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765977.735162:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765978.759065:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765979.783065:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765980.807039:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775765981.830992:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765981.831002:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775765981.831007:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765981.831017:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1870 00000100:00080000:1.0:1775765981.831029:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765981.831035:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765981.831050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775765981.831084:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741915008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765981.831109:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765981.831112:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775765981.831113:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765981.831116:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1870 00000100:00080000:1.0:1775765981.831119:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765981.831122:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765981.831129:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765981.831140:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741915136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765981.831153:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765981.831156:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775765981.831158:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765981.831160:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1870 00000100:00080000:1.0:1775765981.831162:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765981.831164:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765981.831167:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775765981.831174:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741915264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775765981.831182:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775765981.831185:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775765981.831186:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775765981.831190:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1870 00000100:00080000:1.0:1775765981.831192:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775765981.831195:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775765981.831199:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765981.831204:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741915008:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775765981.831210:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741915392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765981.831267:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765981.831302:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765981/real 1775765981] req@ffff90c203494e00 x1862023741915008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765981.831319:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741915008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765981.831337:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765981.831341:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765981.831344:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741915008:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765981.831351:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765981.831367:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765981.831376:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765981/real 1775765981] req@ffff90c203497480 x1862023741915136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765981.831385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741915136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765981.831392:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765981.831394:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765981.831397:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765981.831401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765981.831412:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765981.831418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765981/real 1775765981] req@ffff90c203494000 x1862023741915264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765981.831426:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741915264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765981.831437:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765981.831439:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765981.831442:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765981.831471:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915392:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765981.831484:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765981.831491:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765981/real 1775765981] req@ffff90c203495f80 x1862023741915392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765981.831500:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741915392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766006 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765981.831508:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765981.831510:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765981.831512:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915392:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765982.855150:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765983.879068:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765984.903049:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765985.927019:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775765986.951042:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765986.951051:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775765986.951057:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765986.951065:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1875 00000100:00080000:2.0:1775765986.951078:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765986.951085:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765986.951125:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd500 x1862023741915520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765986.951161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765986.951165:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775765986.951167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765986.951170:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1875 00000100:00080000:2.0:1775765986.951173:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765986.951175:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765986.951184:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765986.951194:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bfb80 x1862023741915648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765986.951204:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765986.951207:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775765986.951208:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765986.951212:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1875 00000100:00080000:2.0:1775765986.951215:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765986.951218:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765986.951221:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775765986.951230:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023741915776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775765986.951239:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775765986.951245:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1775765986.951250:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741915520:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775765986.951250:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775765986.951258:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1875 00000100:00080000:2.0:1775765986.951266:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775765986.951270:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775765986.951274:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765986.951285:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023741915904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775765986.951355:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00010000:00010000:3.0:1775765986.951387:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765986.951402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765986/real 1775765986] req@ffff90c2034bd500 x1862023741915520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765986.951420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd500 x1862023741915520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765986.951431:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765986.951434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765986.951437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741915520:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775765986.951444:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915648:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765986.951462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765986.951474:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765986/real 1775765986] req@ffff90c2034bfb80 x1862023741915648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765986.951486:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bfb80 x1862023741915648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765986.951495:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765986.951499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765986.951502:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765986.951533:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915776:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775765986.951550:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765986.951559:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765986/real 1775765986] req@ffff90c2034be300 x1862023741915776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765986.951567:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023741915776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765986.951573:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765986.951575:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765986.951578:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775765986.951582:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915904:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775765986.951593:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775765986.951598:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765986/real 1775765986] req@ffff90c2034bf480 x1862023741915904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765986.951606:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023741915904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766011 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775765986.951614:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775765986.951616:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775765986.951618:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741915904:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765987.975018:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765988.999086:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765990.023049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765991.047114:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765992.070965:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765992.070975:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765992.070980:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765992.070990:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1880 00000100:00080000:0.0:1775765992.071003:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765992.071011:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765992.071022:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765992.071049:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741916032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765992.071081:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765992.071085:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765992.071098:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765992.071104:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1880 00000100:00080000:0.0:1775765992.071108:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765992.071112:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775765992.071121:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741916032:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775765992.071122:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765992.071140:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741916160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765992.071151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765992.071166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765992.071168:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765992.071172:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1880 00000100:00080000:0.0:1775765992.071176:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765992.071179:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765992.071183:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765992.071193:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741916288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765992.071196:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775765992.071208:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765992.071212:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765992.071214:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765992.071220:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1880 00000100:00080000:0.0:1775765992.071225:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765992.071229:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765992.071234:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775765992.071234:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765992/real 1775765992] req@ffff90c203638700 x1862023741916032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775765992.071248:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741916416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765992.071254:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741916032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765992.071271:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765992.071275:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765992.071278:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741916032:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765992.071307:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765992.071321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765992.071329:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765992/real 1775765992] req@ffff90c203639500 x1862023741916160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765992.071336:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741916160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765992.071342:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765992.071344:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765992.071346:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765992.071350:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765992.071368:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765992.071374:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765992/real 1775765992] req@ffff90c203639180 x1862023741916288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765992.071381:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741916288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765992.071386:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765992.071388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765992.071389:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765992.071393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916416:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765992.071402:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765992.071407:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765992/real 1775765992] req@ffff90c203638000 x1862023741916416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765992.071431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741916416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765992.071438:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765992.071440:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765992.071442:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916416:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765993.095102:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765994.119021:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765995.143063:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765996.167181:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775765997.191046:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765997.191059:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775765997.191064:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765997.191078:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1885 00000100:00080000:0.0:1775765997.191094:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765997.191104:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775765997.191118:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775765997.191158:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741916544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765997.191187:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765997.191190:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775765997.191193:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765997.191197:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1885 00000100:00080000:0.0:1775765997.191201:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765997.191206:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765997.191218:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765997.191237:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741916672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775765997.191243:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741916544:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775765997.191258:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765997.191262:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775765997.191264:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765997.191268:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1885 00000100:00080000:0.0:1775765997.191272:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765997.191277:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765997.191283:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775765997.191295:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741916800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775765997.191302:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775765997.191305:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775765997.191306:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775765997.191309:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1885 00000100:00080000:0.0:1775765997.191312:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775765997.191315:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775765997.191318:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775765997.191327:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741916928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775765997.191331:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765997.191386:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765997/real 1775765997] req@ffff90c20363aa00 x1862023741916544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765997.191419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741916544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765997.191428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765997.191435:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765997.191441:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741916544:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775765997.191464:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765997.191483:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765997.191491:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765997/real 1775765997] req@ffff90c20363b100 x1862023741916672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765997.191499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741916672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765997.191505:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765997.191508:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765997.191510:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765997.191513:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775765997.191524:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765997.191529:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765997/real 1775765997] req@ffff90c20363bb80 x1862023741916800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765997.191536:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741916800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765997.191546:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765997.191548:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765997.191550:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775765997.191554:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916928:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775765997.191565:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775765997.191570:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775765997/real 1775765997] req@ffff90c20363a680 x1862023741916928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775765997.191576:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741916928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775765997.191583:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775765997.191585:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775765997.191587:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741916928:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775765998.215112:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775765999.239115:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766000.263175:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766001.287159:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766002.311018:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766002.311027:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766002.311031:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766002.311040:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1890 00000100:00080000:3.0:1775766002.311050:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766002.311057:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766002.311084:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766002.311105:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741917056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766002.311129:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766002.311132:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766002.311133:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766002.311135:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1890 00000100:00080000:3.0:1775766002.311138:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766002.311141:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766002.311149:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766002.311159:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741917184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766002.311174:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741917056:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766002.311182:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766002.311184:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766002.311185:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766002.311188:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1890 00000100:00080000:3.0:1775766002.311190:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766002.311192:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766002.311196:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766002.311203:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741917312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766002.311209:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766002.311211:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766002.311212:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766002.311215:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1890 00000100:00080000:3.0:1775766002.311229:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766002.311231:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766002.311235:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766002.311239:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766002.311245:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741917440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766002.311276:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766002/real 1775766002] req@ffff90c218da0700 x1862023741917056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766002.311290:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741917056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766002.311298:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766002.311300:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766002.311303:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741917056:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766002.311323:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917184:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766002.311338:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766002.311345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766002/real 1775766002] req@ffff90c218da3480 x1862023741917184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766002.311361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741917184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766002.311367:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766002.311369:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766002.311371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766002.311375:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766002.311386:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766002.311392:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766002/real 1775766002] req@ffff90c218da0380 x1862023741917312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766002.311399:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741917312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766002.311404:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766002.311406:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766002.311408:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766002.311411:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917440:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766002.311421:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766002.311426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766002/real 1775766002] req@ffff90c218da0e00 x1862023741917440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766002.311433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741917440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766002.311439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766002.311441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766002.311443:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917440:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766003.335070:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766004.359092:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766005.383049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766006.406996:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766007.430958:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766007.430966:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766007.430971:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766007.430979:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1895 00000100:00080000:0.0:1775766007.430989:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766007.430994:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766007.430996:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766007.431036:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741917568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766007.431068:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766007.431072:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766007.431075:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766007.431079:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1895 00000100:00080000:0.0:1775766007.431084:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766007.431089:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766007.431099:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766007.431105:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741917568:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766007.431120:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741917696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766007.431133:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766007.431138:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766007.431140:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766007.431145:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1895 00000100:00080000:0.0:1775766007.431149:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766007.431154:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766007.431159:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766007.431163:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766007.431172:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741917824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766007.431180:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766007.431183:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766007.431184:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766007.431188:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1895 00000100:00080000:0.0:1775766007.431192:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766007.431196:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766007.431200:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766007/real 1775766007] req@ffff90c203638380 x1862023741917568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1775766007.431202:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766007.431219:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741917952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766007.431222:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741917568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766007.431259:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766007.431264:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766007.431268:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741917568:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766007.431289:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766007.431303:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766007.431322:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766007/real 1775766007] req@ffff90c203639f80 x1862023741917696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766007.431329:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741917696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766007.431340:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766007.431342:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766007.431344:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766007.431348:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766007.431357:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766007.431362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766007/real 1775766007] req@ffff90c203638e00 x1862023741917824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766007.431369:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741917824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766007.431385:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766007.431386:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766007.431388:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766007.431391:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917952:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766007.431400:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766007.431405:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766007/real 1775766007] req@ffff90c20363b480 x1862023741917952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766007.431411:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741917952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766007.431418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766007.431420:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766007.431423:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741917952:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766008.455056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766009.479107:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766010.503231:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766011.527079:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766012.551002:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766012.551012:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766012.551016:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766012.551025:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1901 00000100:00080000:3.0:1775766012.551037:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766012.551045:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766012.551049:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766012.551090:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741918080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766012.551134:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766012.551138:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766012.551140:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766012.551143:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1901 00000100:00080000:3.0:1775766012.551147:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766012.551151:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766012.551160:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766012.551172:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741918080:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766012.551175:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741918208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766012.551199:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766012.551202:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766012.551204:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766012.551207:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1901 00000100:00080000:3.0:1775766012.551210:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766012.551212:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766012.551216:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766012.551224:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741918336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766012.551231:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766012.551234:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766012.551235:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766012.551239:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1901 00000100:00080000:3.0:1775766012.551254:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766012.551257:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766012.551261:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766012.551266:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766012.551275:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741918464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766012.551315:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766012/real 1775766012] req@ffff90c218da2d80 x1862023741918080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766012.551333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741918080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766012.551342:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766012.551345:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766012.551349:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741918080:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766012.551389:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918208:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766012.551410:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766012.551421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766012/real 1775766012] req@ffff90c218da3800 x1862023741918208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766012.551432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741918208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766012.551439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766012.551442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766012.551445:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766012.551450:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918336:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766012.551462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766012.551470:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766012/real 1775766012] req@ffff90c218da1880 x1862023741918336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766012.551479:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741918336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766012.551486:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766012.551488:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766012.551490:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766012.551495:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918464:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766012.551507:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766012.551530:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766012/real 1775766012] req@ffff90c218da0e00 x1862023741918464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766012.551538:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741918464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766012.551545:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766012.551547:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766012.551549:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918464:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766013.575116:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766014.599088:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766015.623095:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766016.647115:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766017.670988:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775766017.670998:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766017.670999:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766017.671006:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766017.671016:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1906 00000100:00080000:1.0:1775766017.671027:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766017.671035:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766017.671074:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741918592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766017.671098:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766017.671101:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766017.671102:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766017.671105:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1906 00000100:00080000:1.0:1775766017.671108:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766017.671112:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766017.671120:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766017.671133:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741918720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766017.671134:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741918592:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766017.671148:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766017.671150:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766017.671151:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766017.671153:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1906 00000100:00080000:1.0:1775766017.671155:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766017.671158:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766017.671161:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766017.671168:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741918848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766017.671179:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766017.671181:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766017.671182:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766017.671185:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1906 00000100:00080000:1.0:1775766017.671187:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766017.671190:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766017.671193:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766017.671201:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766017.671202:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741918976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766017.671244:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766017/real 1775766017] req@ffff90c203496d80 x1862023741918592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766017.671266:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741918592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766017.671277:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766017.671280:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766017.671283:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741918592:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766017.671302:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766017.671321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766017.671334:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766017/real 1775766017] req@ffff90c203494380 x1862023741918720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766017.671347:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741918720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766017.671357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766017.671361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766017.671365:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766017.671372:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766017.671390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766017.671398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766017/real 1775766017] req@ffff90c203497100 x1862023741918848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766017.671407:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741918848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766017.671412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766017.671415:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766017.671417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766017.671421:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918976:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766017.671432:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766017.671437:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766017/real 1775766017] req@ffff90c203495880 x1862023741918976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766042 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766017.671445:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741918976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766042 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766017.671451:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766017.671453:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766017.671455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741918976:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766018.695048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766019.719170:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766020.743042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766021.767075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766022.791002:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766022.791011:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766022.791016:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766022.791024:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1911 00000100:00080000:0.0:1775766022.791036:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766022.791042:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766022.791084:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741919104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766022.791111:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766022.791115:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766022.791117:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766022.791122:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1911 00000100:00080000:0.0:1775766022.791126:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766022.791131:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766022.791137:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00100000:0.0:1775766022.791141:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766022.791158:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741919104:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766022.791161:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741919232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766022.791183:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766022.791185:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766022.791186:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766022.791191:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1911 00000100:00080000:0.0:1775766022.791194:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766022.791198:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766022.791207:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766022.791218:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741919360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766022.791225:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766022.791228:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766022.791230:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766022.791233:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1911 00000100:00080000:0.0:1775766022.791238:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766022.791242:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766022.791245:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766022.791251:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741919488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766022.791278:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766022.791335:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766022/real 1775766022] req@ffff90c20363b800 x1862023741919104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766022.791362:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741919104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766022.791378:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766022.791384:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766022.791390:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741919104:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766022.791423:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766022.791449:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766022.791463:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766022/real 1775766022] req@ffff90c20363b480 x1862023741919232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766022.791479:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741919232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766022.791489:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766022.791519:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766022.791524:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766022.791532:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766022.791555:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766022.791568:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766022/real 1775766022] req@ffff90c203638e00 x1862023741919360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766022.791582:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741919360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766022.791592:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766022.791596:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766022.791599:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766022.791606:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919488:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766022.791625:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766022.791634:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766022/real 1775766022] req@ffff90c203639f80 x1862023741919488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766047 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766022.791647:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741919488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766047 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766022.791668:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766022.791672:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766022.791676:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919488:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766023.815011:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766024.839026:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766025.863104:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766026.887037:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766027.911011:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766027.911020:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766027.911025:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766027.911037:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1916 00000100:00080000:3.0:1775766027.911049:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775766027.911055:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766027.911057:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766027.911104:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741919616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766027.911131:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766027.911135:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766027.911137:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766027.911142:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1916 00000100:00080000:3.0:1775766027.911148:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766027.911153:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766027.911162:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741919616:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775766027.911164:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766027.911183:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741919744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766027.911192:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766027.911195:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766027.911196:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766027.911200:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1916 00000100:00080000:3.0:1775766027.911212:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766027.911215:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766027.911219:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766027.911231:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741919872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766027.911243:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766027.911246:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766027.911248:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766027.911253:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1916 00000100:00100000:2.0:1775766027.911257:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919744:192.168.202.155@tcp:38:kworker.0 00000100:00080000:3.0:1775766027.911257:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766027.911266:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766027.911272:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766027.911286:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741920000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766027.911287:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766027.911313:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920000:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766028.935044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766029.959096:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766030.983189:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775766030.983360:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775766030.983462:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766030.983486:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766030.983536:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766030.983549:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766030.983559:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766030.983614:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766027/real 1775766030] req@ffff90c218da0a80 x1862023741920000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766030.983652:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741920000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766030.983673:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766030.983681:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766030.983688:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920000:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766030.983699:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766027/real 1775766030] req@ffff90c218da3480 x1862023741919744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766030.983724:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741919744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766030.983737:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766030.983743:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766030.983748:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766030.983758:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766027/real 1775766030] req@ffff90c218da0700 x1862023741919872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766030.983776:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741919872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766030.983788:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766030.983792:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766030.983796:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741919872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766030.983805:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766027/real 1775766030] req@ffff90c218da0380 x1862023741919616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766030.983823:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741919616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766052 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766030.983833:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766030.983837:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766030.983841:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741919616:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766032.007096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766033.031055:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775766033.031058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766033.031067:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766033.031072:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766033.031084:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1921 00000100:00080000:0.0:1775766033.031096:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766033.031104:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766033.031141:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741920128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766033.031180:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766033.031184:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766033.031186:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766033.031191:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1921 00000100:00080000:0.0:1775766033.031196:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766033.031202:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766033.031213:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766033.031234:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741920256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766033.031245:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741920128:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766033.031254:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766033.031268:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766033.031270:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766033.031276:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1921 00000100:00080000:0.0:1775766033.031280:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766033.031285:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766033.031290:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766033.031308:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741920384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766033.031325:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766033.031329:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766033.031330:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766033.031335:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1921 00000100:00080000:0.0:1775766033.031339:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766033.031343:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766033.031347:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766033.031358:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741920512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766033.031447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766033.031522:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766033/real 1775766033] req@ffff90c203638380 x1862023741920128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766033.031555:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741920128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766033.031576:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766033.031584:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766033.031592:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741920128:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766033.031656:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766033.031694:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766033.031714:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766033/real 1775766033] req@ffff90c20363a680 x1862023741920256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766033.031732:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741920256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766033.031745:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766033.031750:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766033.031755:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766033.031765:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766033.031797:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766033.031814:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766033/real 1775766033] req@ffff90c20363bb80 x1862023741920384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766033.031833:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741920384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766033.031846:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766033.031852:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766033.031857:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766033.031868:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920512:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766033.031963:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766033.031980:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766033/real 1775766033] req@ffff90c20363b100 x1862023741920512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766033.031998:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741920512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766033.032012:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766033.032017:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766033.032021:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920512:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766034.055070:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766035.079072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766036.103029:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766037.127119:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766038.151006:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766038.151015:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766038.151022:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766038.151033:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1926 00000100:00080000:3.0:1775766038.151047:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766038.151056:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766038.151110:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741920640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766038.151137:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766038.151140:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766038.151142:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766038.151147:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1926 00010000:00010000:1.0:1775766038.151150:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766038.151151:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766038.151156:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766038.151166:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766038.151180:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741920768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766038.151190:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741920640:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766038.151192:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766038.151199:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766038.151202:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766038.151206:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1926 00000100:00080000:3.0:1775766038.151209:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766038.151212:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766038.151224:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766038.151234:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741920896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766038.151248:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766038.151251:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766038.151252:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766038.151255:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1926 00000100:00080000:3.0:1775766038.151267:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766038.151269:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766038.151273:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766038.151279:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766038.151286:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741921024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766038.151339:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766038/real 1775766038] req@ffff90c218da3100 x1862023741920640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766038.151378:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741920640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766038.151392:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766038.151396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766038.151401:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741920640:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766038.151428:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766038.151455:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766038.151470:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766038/real 1775766038] req@ffff90c218da1f80 x1862023741920768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766038.151486:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741920768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766038.151496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766038.151500:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766038.151504:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766038.151511:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920896:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766038.151534:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766038.151547:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766038/real 1775766038] req@ffff90c218da0000 x1862023741920896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766038.151562:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741920896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766038.151571:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766038.151575:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766038.151579:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741920896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766038.151586:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921024:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766038.151608:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766038.151621:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766038/real 1775766038] req@ffff90c218da0380 x1862023741921024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766038.151635:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741921024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766038.151654:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766038.151658:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766038.151661:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921024:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766039.175034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766040.199067:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766041.223025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766042.247076:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766043.271065:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766043.271074:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766043.271079:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766043.271090:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1931 00000100:00080000:0.0:1775766043.271103:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766043.271107:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766043.271111:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766043.271154:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741921152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766043.271184:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766043.271187:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766043.271189:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766043.271193:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1931 00000100:00080000:0.0:1775766043.271196:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766043.271199:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766043.271209:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766043.271221:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741921280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766043.271244:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766043.271247:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775766043.271258:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741921152:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766043.271259:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766043.271267:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1931 00000100:00080000:0.0:1775766043.271272:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766043.271275:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766043.271279:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766043.271291:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741921408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766043.271343:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766043.271346:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766043.271348:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766043.271352:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1931 00000100:00080000:0.0:1775766043.271356:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766043.271359:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766043.271365:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766043.271377:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741921536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766043.271383:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766043.271439:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766043/real 1775766043] req@ffff90c20363aa00 x1862023741921152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766043.271474:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741921152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766043.271488:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766043.271493:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766043.271498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741921152:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766043.271530:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766043.271564:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766043.271581:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766043/real 1775766043] req@ffff90c203638000 x1862023741921280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766043.271599:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741921280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766043.271612:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766043.271616:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766043.271635:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766043.271646:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766043.271674:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766043.271690:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766043/real 1775766043] req@ffff90c203639180 x1862023741921408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766043.271707:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741921408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766043.271717:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766043.271722:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766043.271727:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766043.271735:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921536:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766043.271763:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766043.271778:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766043/real 1775766043] req@ffff90c203639500 x1862023741921536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766043.271793:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741921536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766043.271807:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766043.271811:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766043.271815:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921536:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766044.295029:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766045.319111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766046.343061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766047.367065:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766048.390968:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766048.390978:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766048.390984:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766048.390995:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1936 00000100:00080000:3.0:1775766048.391007:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766048.391015:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766048.391022:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766048.391056:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741921664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766048.391078:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766048.391081:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766048.391083:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766048.391086:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1936 00000100:00080000:3.0:1775766048.391089:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766048.391093:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766048.391100:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766048.391110:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741921664:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766048.391111:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741921792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766048.391124:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766048.391127:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766048.391128:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766048.391131:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1936 00000100:00080000:3.0:1775766048.391133:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766048.391135:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766048.391138:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766048.391146:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741921920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766048.391153:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766048.391155:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766048.391156:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766048.391159:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1936 00000100:00080000:3.0:1775766048.391164:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766048.391166:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766048.391168:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766048.391174:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766048.391179:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741922048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766048.391210:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766048/real 1775766048] req@ffff90c218da0700 x1862023741921664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766048.391225:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741921664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766048.391233:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766048.391236:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766048.391239:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741921664:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766048.391281:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766048.391297:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766048.391306:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766048/real 1775766048] req@ffff90c218da3480 x1862023741921792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766048.391314:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741921792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766048.391319:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766048.391322:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766048.391324:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766048.391327:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766048.391337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766048.391343:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766048/real 1775766048] req@ffff90c218da0a80 x1862023741921920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766048.391350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741921920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766048.391359:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766048.391361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766048.391363:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741921920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766048.391366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922048:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766048.391375:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766048.391380:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766048/real 1775766048] req@ffff90c218da0e00 x1862023741922048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766048.391386:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741922048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766048.391393:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766048.391395:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766048.391396:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922048:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766049.415014:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766050.439077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766051.463052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766052.487059:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766053.510986:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766053.510995:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766053.511000:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766053.511008:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1941 00000100:00080000:0.0:1775766053.511030:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766053.511036:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766053.511038:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766053.511070:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741922176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766053.511094:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766053.511097:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766053.511099:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766053.511101:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1941 00000100:00080000:0.0:1775766053.511104:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766053.511106:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766053.511114:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766053.511127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741922304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766053.511128:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741922176:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766053.511141:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766053.511143:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766053.511145:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766053.511147:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1941 00000100:00080000:0.0:1775766053.511149:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766053.511152:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766053.511159:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766053.511171:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741922432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766053.511182:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766053.511185:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766053.511187:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766053.511192:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1941 00000100:00080000:0.0:1775766053.511210:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766053.511216:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766053.511221:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:0.0:1775766053.511222:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766053.511233:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741922560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766053.511267:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766053/real 1775766053] req@ffff90c203638700 x1862023741922176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766053.511282:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741922176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766053.511290:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766053.511293:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766053.511296:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741922176:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766053.511316:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922304:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766053.511333:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766053.511341:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766053/real 1775766053] req@ffff90c203639500 x1862023741922304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766053.511349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741922304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766053.511354:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766053.511357:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766053.511359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766053.511362:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922432:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766053.511371:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766053.511377:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766053/real 1775766053] req@ffff90c203639180 x1862023741922432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766053.511384:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741922432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766053.511389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766053.511390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766053.511392:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766053.511395:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922560:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766053.511405:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766053.511410:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766053/real 1775766053] req@ffff90c203638000 x1862023741922560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766053.511418:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741922560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766053.511424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766053.511425:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766053.511427:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922560:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766054.535093:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766055.559098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766056.583237:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766057.607202:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766058.631059:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766058.631076:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766058.631081:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766058.631090:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1947 00000100:00080000:1.0:1775766058.631102:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766058.631109:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766058.631131:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766058.631196:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741922688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766058.631233:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766058.631236:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766058.631238:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766058.631242:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1947 00000100:00080000:1.0:1775766058.631247:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766058.631252:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766058.631262:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766058.631273:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741922688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766058.631279:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741922816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766058.631290:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766058.631293:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766058.631295:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766058.631299:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1947 00000100:00080000:1.0:1775766058.631302:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766058.631304:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766058.631309:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766058.631320:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741922944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766058.631333:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766058.631337:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766058.631339:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775766058.631345:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766058.631349:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1947 00000100:00080000:1.0:1775766058.631355:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766058.631370:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766058.631378:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766058.631396:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741923072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766058.631405:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766058/real 1775766058] req@ffff90c203495c00 x1862023741922688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766058.631425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741922688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766058.631436:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766058.631440:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766058.631444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741922688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766058.631483:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766058.631502:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766058.631512:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766058/real 1775766058] req@ffff90c203497b80 x1862023741922816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766058.631538:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741922816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766058.631551:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766058.631554:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766058.631557:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766058.631562:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766058.631575:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766058.631582:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766058/real 1775766058] req@ffff90c203496300 x1862023741922944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766058.631591:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741922944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766058.631597:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766058.631599:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766058.631601:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741922944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766058.631605:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923072:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766058.631617:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766058.631623:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766058/real 1775766058] req@ffff90c203496a00 x1862023741923072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766083 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766058.631632:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741923072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766083 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766058.631640:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766058.631642:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766058.631646:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923072:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766059.655195:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766060.679082:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766061.703046:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766062.727024:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766063.751016:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766063.751025:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766063.751031:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:1.0:1775766063.751033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766063.751041:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1952 00000100:00080000:3.0:1775766063.751054:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766063.751061:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766063.751117:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741923200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766063.751153:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766063.751158:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766063.751161:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766063.751166:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1952 00000100:00080000:3.0:1775766063.751172:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766063.751177:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766063.751197:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766063.751213:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741923328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766063.751221:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741923200:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766063.751242:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766063.751245:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766063.751247:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766063.751252:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1952 00000100:00080000:3.0:1775766063.751256:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766063.751259:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766063.751264:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766063.751276:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741923456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766063.751285:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766063.751288:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766063.751289:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766063.751292:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1952 00000100:00080000:3.0:1775766063.751299:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766063.751302:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766063.751314:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766063.751315:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766063.751329:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741923584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766063.751373:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766063/real 1775766063] req@ffff90c218da1880 x1862023741923200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766063.751427:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741923200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766063.751442:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766063.751449:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766063.751455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741923200:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766063.751488:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766063.751524:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766063.751542:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766063/real 1775766063] req@ffff90c218da3800 x1862023741923328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766063.751569:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741923328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766063.751581:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766063.751586:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766063.751591:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766063.751600:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766063.751627:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766063.751644:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766063/real 1775766063] req@ffff90c218da2d80 x1862023741923456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766063.751662:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741923456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766063.751673:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766063.751678:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766063.751683:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766063.751692:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923584:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766063.751720:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766063.751736:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766063/real 1775766063] req@ffff90c218da0e00 x1862023741923584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766088 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766063.751754:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741923584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766088 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766063.751769:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766063.751774:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766063.751778:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923584:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766064.775257:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766065.799130:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766066.823107:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766067.846997:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766068.870985:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775766068.870985:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766068.870994:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766068.870998:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766068.871007:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1957 00000100:00080000:1.0:1775766068.871018:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766068.871024:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766068.871083:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741923712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766068.871114:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766068.871118:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766068.871119:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766068.871122:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1957 00000100:00080000:1.0:1775766068.871126:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766068.871130:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766068.871139:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766068.871152:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741923840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766068.871152:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741923712:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766068.871166:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766068.871169:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766068.871170:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766068.871173:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1957 00000100:00080000:1.0:1775766068.871177:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766068.871181:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766068.871187:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766068.871199:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741923968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766068.871210:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766068.871213:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766068.871215:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775766068.871220:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766068.871225:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1957 00000100:00080000:1.0:1775766068.871230:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766068.871236:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766068.871242:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766068.871253:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741924096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766068.871260:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766068/real 1775766068] req@ffff90c203496680 x1862023741923712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766068.871277:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741923712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766068.871286:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766068.871289:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766068.871292:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741923712:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766068.871309:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923840:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766068.871323:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766068.871331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766068/real 1775766068] req@ffff90c203494a80 x1862023741923840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766068.871339:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741923840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766068.871344:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766068.871346:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766068.871348:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766068.871352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766068.871361:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766068.871366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766068/real 1775766068] req@ffff90c203495180 x1862023741923968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766068.871373:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741923968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766068.871381:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766068.871384:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766068.871388:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741923968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766068.871394:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924096:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766068.871415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766068.871426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766068/real 1775766068] req@ffff90c203495500 x1862023741924096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766068.871440:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741924096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766093 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766068.871449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766068.871451:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766068.871453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924096:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766069.895191:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766070.919131:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766071.943182:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766072.967157:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766073.991075:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766073.991084:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766073.991088:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766073.991097:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1962 00000100:00080000:1.0:1775766073.991108:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766073.991115:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766073.991134:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766073.991153:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741924224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766073.991174:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766073.991177:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766073.991178:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766073.991181:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1962 00000100:00080000:1.0:1775766073.991183:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766073.991186:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766073.991194:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766073.991206:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741924352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766073.991211:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741924224:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766073.991216:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766073.991223:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766073.991224:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766073.991227:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1962 00000100:00080000:1.0:1775766073.991229:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766073.991231:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766073.991234:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766073.991241:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741924480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766073.991269:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766073.991272:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766073.991274:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766073.991277:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1962 00000100:00080000:1.0:1775766073.991280:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766073.991283:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766073.991286:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766073.991295:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741924608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766073.991310:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766073.991368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766073/real 1775766073] req@ffff90c203494700 x1862023741924224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766073.991403:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741924224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766073.991413:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766073.991417:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766073.991421:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741924224:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766073.991444:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766073.991466:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766073.991476:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766073/real 1775766073] req@ffff90c203495500 x1862023741924352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766073.991487:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741924352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766073.991495:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766073.991498:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766073.991500:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766073.991505:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766073.991517:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766073.991524:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766073/real 1775766073] req@ffff90c203495180 x1862023741924480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766073.991533:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741924480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766073.991539:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766073.991542:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766073.991544:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766073.991548:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924608:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766073.991560:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766073.991567:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766073/real 1775766073] req@ffff90c203494a80 x1862023741924608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766073.991575:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741924608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766098 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766073.991605:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766073.991608:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766073.991610:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924608:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766075.015152:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766076.039038:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766077.063148:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766078.087115:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766079.111060:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766079.111070:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766079.111075:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766079.111088:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1967 00000100:00080000:1.0:1775766079.111101:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766079.111110:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766079.111113:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766079.111162:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741924736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766079.111188:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766079.111191:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766079.111192:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766079.111195:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1967 00000100:00080000:1.0:1775766079.111199:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766079.111203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766079.111213:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766079.111225:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741924736:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766079.111227:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741924864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766079.111241:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766079.111244:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766079.111245:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766079.111248:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1967 00000100:00080000:1.0:1775766079.111250:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766079.111253:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766079.111257:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766079.111268:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741924992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766079.111276:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766079.111278:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766079.111280:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766079.111283:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1967 00000100:00080000:1.0:1775766079.111286:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766079.111289:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766079.111292:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766079.111294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766079.111305:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741925120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766079.111336:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766079/real 1775766079] req@ffff90c203496680 x1862023741924736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766079.111352:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741924736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766079.111360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766079.111363:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766079.111367:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741924736:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766079.111396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924864:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766079.111412:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766079.111421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766079/real 1775766079] req@ffff90c203496a00 x1862023741924864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766079.111429:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741924864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766079.111435:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766079.111438:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766079.111440:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766079.111446:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924992:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766079.111457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766079.111463:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766079/real 1775766079] req@ffff90c203496300 x1862023741924992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766079.111472:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741924992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766079.111477:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766079.111479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766079.111481:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741924992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766079.111485:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925120:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766079.111495:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766079.111501:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766079/real 1775766079] req@ffff90c203497b80 x1862023741925120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766079.111508:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741925120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766079.111515:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766079.111517:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766079.111519:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925120:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766080.135123:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766081.159148:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766082.183110:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766083.207100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766084.231117:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766084.231133:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766084.231141:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766084.231157:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1972 00010000:00010000:3.0:1775766084.231161:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766084.231173:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766084.231184:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766084.231249:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023741925248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766084.231303:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766084.231311:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766084.231315:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766084.231323:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1972 00000100:00080000:2.0:1775766084.231332:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766084.231341:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766084.231373:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766084.231385:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741925248:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766084.231402:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023741925376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766084.231420:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766084.231425:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766084.231429:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766084.231436:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1972 00000100:00080000:2.0:1775766084.231446:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766084.231451:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766084.231459:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766084.231471:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766084.231478:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023741925504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766084.231502:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766084.231506:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766084.231510:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766084.231518:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1972 00000100:00080000:2.0:1775766084.231524:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766084.231532:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766084/real 1775766084] req@ffff90c2034bea00 x1862023741925248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766084.231536:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766084.231547:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766084.231571:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023741925632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766084.231587:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023741925248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766084.231606:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766084.231617:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766084.231621:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741925248:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766084.231648:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766084.231674:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766084.231685:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766084/real 1775766084] req@ffff90c2034bca80 x1862023741925376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766084.231695:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023741925376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766084.231701:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766084.231704:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766084.231706:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766084.231711:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766084.231724:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766084.231730:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766084/real 1775766084] req@ffff90c2034bce00 x1862023741925504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766084.231738:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023741925504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766084.231744:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766084.231747:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766084.231749:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766084.231753:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925632:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766084.231764:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766084.231770:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766084/real 1775766084] req@ffff90c2034bc380 x1862023741925632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766084.231784:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023741925632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766084.231791:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766084.231794:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766084.231796:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925632:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766085.255108:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766086.279144:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766087.303109:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766088.327113:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766089.351002:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766089.351014:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766089.351019:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766089.351031:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1977 00000100:00080000:3.0:1775766089.351044:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766089.351051:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766089.351072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766089.351098:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741925760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766089.351123:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766089.351126:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766089.351127:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766089.351131:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1977 00000100:00080000:3.0:1775766089.351134:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766089.351137:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766089.351147:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766089.351157:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741925888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766089.351162:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741925760:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766089.351166:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766089.351172:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766089.351174:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766089.351178:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1977 00000100:00080000:3.0:1775766089.351180:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766089.351182:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766089.351185:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766089.351193:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741926016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766089.351199:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766089.351201:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766089.351202:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766089.351206:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1977 00000100:00080000:3.0:1775766089.351209:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766089.351211:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766089.351215:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766089.351224:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741926144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766089.351253:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766089.351313:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766089/real 1775766089] req@ffff90c218da0a80 x1862023741925760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766089.351336:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741925760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766089.351348:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766089.351354:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766089.351358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741925760:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766089.351388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766089.351413:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766089.351427:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766089/real 1775766089] req@ffff90c218da3480 x1862023741925888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766089.351440:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741925888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766089.351455:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766089.351459:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766089.351463:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741925888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766089.351481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766089.351506:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766089.351519:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766089/real 1775766089] req@ffff90c218da0700 x1862023741926016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766089.351531:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741926016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766089.351540:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766089.351544:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766089.351548:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766089.351554:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926144:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766089.351577:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766089.351588:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766089/real 1775766089] req@ffff90c218da0380 x1862023741926144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766089.351612:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741926144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766089.351624:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766089.351628:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766089.351633:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926144:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766090.375132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766091.399136:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766092.423052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766093.447076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766094.470983:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766094.470996:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766094.471020:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766094.471032:0:7481:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1982 00000100:00080000:3.0:1775766094.471044:0:7481:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766094.471054:0:7481:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766094.471062:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766094.471100:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741926272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766094.471125:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766094.471129:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766094.471130:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766094.471133:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1982 00000100:00080000:3.0:1775766094.471136:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766094.471138:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766094.471147:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766094.471156:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741926400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766094.471164:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766094.471167:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766094.471169:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766094.471173:0:7481:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1982 00000100:00080000:3.0:1775766094.471176:0:7481:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766094.471178:0:7481:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766094.471182:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766094.471189:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741926528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766094.471197:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741926272:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766094.471198:0:7481:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766094.471216:0:7481:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766094.471218:0:7481:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766094.471222:0:7481:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1982 00000100:00080000:3.0:1775766094.471224:0:7481:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766094.471227:0:7481:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766094.471230:0:7481:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766094.471241:0:7481:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741926656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766094.471379:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766094.471424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766094.471455:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926656:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766095.495064:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766096.519089:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766097.543169:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775766097.543316:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775766097.543382:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766097.543412:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766097.543481:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766097.543502:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766097.543519:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775766097.543611:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766094/real 1775766097] req@ffff90c218da3100 x1862023741926528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766097.543666:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741926528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766097.543687:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766097.543697:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766097.543705:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766097.543723:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766094/real 1775766097] req@ffff90c218da0380 x1862023741926656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766097.543765:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741926656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766097.543810:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766097.543817:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766097.543825:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926656:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775766097.543842:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766094/real 1775766097] req@ffff90c218da0000 x1862023741926272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766097.543868:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741926272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766097.543979:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766097.543987:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766097.543994:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741926272:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766097.544010:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766094/real 1775766097] req@ffff90c218da1f80 x1862023741926400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766097.544037:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741926400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766097.544053:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766097.544059:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766097.544065:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926400:192.168.202.155@tcp:38:kworker.0 00010000:00010000:0.0:1775766098.567023:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766099.591037:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766099.591050:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766099.591060:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766099.591074:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1988 00010000:00010000:1.0:1775766099.591084:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766099.591092:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766099.591105:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766099.591161:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741926784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766099.591203:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766099.591210:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766099.591214:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766099.591222:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1988 00000100:00080000:3.0:1775766099.591228:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766099.591250:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766099.591267:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766099.591296:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741926912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766099.591330:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766099.591338:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766099.591341:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766099.591349:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1988 00000100:00100000:0.0:1775766099.591360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741926784:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766099.591360:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766099.591379:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766099.591391:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766099.591421:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741927040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766099.591459:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775766099.591468:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766099.591475:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766099.591480:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766099.591489:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1988 00000100:00080000:3.0:1775766099.591499:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766099.591506:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766099.591515:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766099.591536:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741927168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766099.591554:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766099/real 1775766099] req@ffff90c218da0700 x1862023741926784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766099.591614:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741926784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766099.591646:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766099.591654:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766099.591664:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741926784:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766099.591678:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766099.591740:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766099.591823:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766099/real 1775766099] req@ffff90c218da3480 x1862023741926912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766099.591850:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741926912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766099.591867:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766099.591939:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766099.591947:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741926912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766099.591991:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766099.592038:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766099.592060:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766099/real 1775766099] req@ffff90c218da0a80 x1862023741927040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766099.592085:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741927040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766099.592100:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766099.592106:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766099.592113:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766099.592126:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927168:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766099.592168:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766099.592191:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766099/real 1775766099] req@ffff90c218da0e00 x1862023741927168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766099.592212:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741927168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766099.592230:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766099.592236:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766099.592241:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927168:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766100.615112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766101.639058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766102.663077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766103.687099:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766104.710979:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766104.710991:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766104.710997:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766104.711010:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1993 00000100:00080000:0.0:1775766104.711024:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766104.711032:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766104.711049:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766104.711073:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741927296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766104.711105:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766104.711110:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766104.711114:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766104.711122:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1993 00000100:00080000:0.0:1775766104.711127:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766104.711133:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766104.711144:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766104.711154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741927296:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766104.711159:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741927424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766104.711173:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766104.711177:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766104.711179:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766104.711183:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1993 00000100:00080000:0.0:1775766104.711191:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766104.711196:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766104.711203:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766104.711216:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741927552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766104.711225:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766104.711227:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766104.711231:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766104.711233:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766104.711237:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1993 00000100:00080000:0.0:1775766104.711241:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766104.711245:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766104.711250:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766104.711260:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741927680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766104.711281:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766104/real 1775766104] req@ffff90c20363aa00 x1862023741927296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766104.711304:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741927296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766104.711322:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766104.711327:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766104.711330:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741927296:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766104.711360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927424:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766104.711382:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766104.711393:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766104/real 1775766104] req@ffff90c20363b100 x1862023741927424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766104.711405:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741927424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766104.711412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766104.711414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766104.711417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766104.711421:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927552:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766104.711434:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766104.711441:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766104/real 1775766104] req@ffff90c20363bb80 x1862023741927552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766104.711450:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741927552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766104.711456:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766104.711458:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766104.711460:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766104.711464:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927680:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766104.711476:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766104.711482:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766104/real 1775766104] req@ffff90c20363a680 x1862023741927680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766129 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766104.711491:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741927680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766129 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766104.711503:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766104.711506:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766104.711509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927680:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766105.735047:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766106.758999:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766107.783051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766108.807027:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766109.831022:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766109.831030:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766109.831034:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766109.831045:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 1998 00000100:00080000:1.0:1775766109.831058:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766109.831062:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766109.831065:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766109.831110:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741927808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766109.831136:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766109.831138:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766109.831140:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766109.831143:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1998 00000100:00080000:1.0:1775766109.831145:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766109.831147:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766109.831156:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766109.831165:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741927936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766109.831174:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766109.831178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766109.831181:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766109.831182:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741927808:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766109.831191:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1998 00000100:00080000:1.0:1775766109.831197:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766109.831200:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766109.831203:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766109.831217:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741928064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766109.831230:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766109.831233:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766109.831235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766109.831238:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 1998 00000100:00080000:1.0:1775766109.831243:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766109.831246:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766109.831250:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766109.831256:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741928192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766109.831271:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766109.831314:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766109/real 1775766109] req@ffff90c203495c00 x1862023741927808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766109.831330:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741927808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766109.831340:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766109.831344:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766109.831347:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741927808:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766109.831370:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927936:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766109.831385:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766109.831393:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766109/real 1775766109] req@ffff90c203495880 x1862023741927936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766109.831400:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741927936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766109.831406:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766109.831409:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766109.831411:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741927936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766109.831415:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766109.831426:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766109.831431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766109/real 1775766109] req@ffff90c203497100 x1862023741928064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766109.831438:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741928064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766109.831443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766109.831445:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766109.831446:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766109.831450:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928192:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766109.831460:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766109.831465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766109/real 1775766109] req@ffff90c203494380 x1862023741928192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766109.831471:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741928192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766134 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766109.831478:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766109.831480:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766109.831482:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928192:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766110.855039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766111.879089:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766112.903078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766113.927000:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766114.951027:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766114.951038:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766114.951043:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766114.951054:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2003 00010000:00010000:1.0:1775766114.951056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766114.951068:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766114.951077:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766114.951130:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741928320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766114.951178:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766114.951183:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766114.951186:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766114.951191:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2003 00000100:00080000:0.0:1775766114.951197:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766114.951202:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766114.951224:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766114.951226:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741928320:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766114.951245:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741928448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766114.951260:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766114.951264:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766114.951267:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766114.951272:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2003 00000100:00080000:0.0:1775766114.951278:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766114.951295:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766114.951303:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766114.951308:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766114.951324:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741928576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766114.951338:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766114.951342:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766114.951345:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766114.951350:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2003 00000100:00100000:2.0:1775766114.951356:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766114/real 1775766114] req@ffff90c203638380 x1862023741928320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766114.951358:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766114.951370:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766114.951379:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766114.951382:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741928320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766114.951396:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1775766114.951401:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741928704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766114.951406:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766114.951415:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741928320:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766114.951451:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766114.951476:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766114.951487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766114/real 1775766114] req@ffff90c203639f80 x1862023741928448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766114.951497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741928448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766114.951505:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766114.951507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766114.951510:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766114.951526:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766114.951538:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766114.951545:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766114/real 1775766114] req@ffff90c203638e00 x1862023741928576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766114.951553:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741928576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766114.951564:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766114.951567:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766114.951569:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766114.951573:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928704:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766114.951595:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766114.951601:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766114/real 1775766114] req@ffff90c20363b480 x1862023741928704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766114.951609:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741928704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766139 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766114.951653:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766114.951656:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766114.951658:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928704:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766115.975080:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766116.999084:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766118.023129:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766119.047121:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766120.070967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766120.070978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766120.070984:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766120.070995:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2008 00000100:00080000:0.0:1775766120.071007:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766120.071012:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766120.071016:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766120.071060:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741928832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766120.071085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766120.071088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766120.071089:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766120.071093:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2008 00000100:00080000:0.0:1775766120.071097:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766120.071100:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766120.071109:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766120.071118:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741928832:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766120.071122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741928960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766120.071131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766120.071133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766120.071134:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766120.071137:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2008 00000100:00080000:0.0:1775766120.071140:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766120.071142:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766120.071144:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766120.071152:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741929088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766120.071159:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766120.071161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766120.071162:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766120.071164:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2008 00000100:00080000:0.0:1775766120.071166:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766120.071168:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766120.071173:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766120.071179:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741929216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766120.071185:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766120.071223:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766120/real 1775766120] req@ffff90c20363b800 x1862023741928832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766120.071238:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741928832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766120.071245:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766120.071248:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766120.071251:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741928832:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766120.071271:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928960:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766120.071285:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766120.071292:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766120/real 1775766120] req@ffff90c20363b480 x1862023741928960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766120.071300:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741928960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766120.071305:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766120.071307:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766120.071309:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741928960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766120.071313:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929088:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766120.071321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766120.071327:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766120/real 1775766120] req@ffff90c203638e00 x1862023741929088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766120.071334:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741929088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766120.071339:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766120.071340:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766120.071342:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766120.071345:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929216:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766120.071355:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766120.071360:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766120/real 1775766120] req@ffff90c203639f80 x1862023741929216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766120.071368:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741929216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766120.071374:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766120.071376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766120.071378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929216:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766121.095042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766122.119113:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766123.143171:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766124.167055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766125.191045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766125.191061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766125.191068:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766125.191084:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2013 00000100:00080000:0.0:1775766125.191103:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766125.191116:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766125.191156:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766125.191183:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741929344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766125.191237:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766125.191245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766125.191250:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766125.191260:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2013 00000100:00080000:0.0:1775766125.191270:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766125.191279:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766125.191301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741929344:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775766125.191311:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766125.191340:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741929472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766125.191366:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766125.191370:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766125.191373:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766125.191378:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2013 00000100:00080000:0.0:1775766125.191386:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766125.191390:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766125.191399:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766125.191404:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766125.191424:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741929600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766125.191439:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766125.191444:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766125.191446:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766125.191452:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2013 00000100:00080000:0.0:1775766125.191459:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766125.191463:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766125.191470:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766125.191472:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766125/real 1775766125] req@ffff90c203638380 x1862023741929344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766125.191491:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741929728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766125.191579:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741929344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766125.191605:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766125.191615:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766125.191637:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741929344:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766125.191678:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766125.191717:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766125.191735:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766125/real 1775766125] req@ffff90c20363a680 x1862023741929472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766125.191751:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741929472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766125.191770:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766125.191774:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766125.191779:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766125.191786:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766125.191805:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766125.191816:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766125/real 1775766125] req@ffff90c20363bb80 x1862023741929600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766125.191829:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741929600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766125.191838:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766125.191842:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766125.191845:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766125.191852:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929728:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766125.191910:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766125.191932:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766125/real 1775766125] req@ffff90c20363b100 x1862023741929728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766125.191960:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741929728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766125.191981:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766125.191988:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766125.191997:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929728:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766126.215129:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766127.239037:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766128.263084:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766129.287043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766130.311020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766130.311029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766130.311033:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766130.311042:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2018 00000100:00080000:3.0:1775766130.311054:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766130.311060:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766130.311063:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766130.311096:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741929856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766130.311122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766130.311124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766130.311126:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766130.311128:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2018 00000100:00080000:3.0:1775766130.311131:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766130.311134:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766130.311151:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766130.311161:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741929984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766130.311165:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741929856:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766130.311174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766130.311177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766130.311178:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766130.311182:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2018 00000100:00080000:3.0:1775766130.311184:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766130.311186:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766130.311189:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766130.311199:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741930112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766130.311206:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766130.311208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766130.311209:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766130.311212:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2018 00000100:00080000:3.0:1775766130.311217:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766130.311219:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766130.311222:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766130.311227:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766130.311231:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741930240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766130.311263:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766130/real 1775766130] req@ffff90c218da2d80 x1862023741929856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766130.311278:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741929856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766130.311285:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766130.311289:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766130.311292:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741929856:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766130.311310:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766130.311324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766130.311331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766130/real 1775766130] req@ffff90c218da3800 x1862023741929984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766130.311339:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741929984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766130.311345:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766130.311347:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766130.311349:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741929984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766130.311353:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766130.311369:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766130.311376:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766130/real 1775766130] req@ffff90c218da1880 x1862023741930112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766130.311384:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741930112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766130.311389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766130.311390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766130.311392:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766130.311396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930240:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766130.311406:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766130.311411:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766130/real 1775766130] req@ffff90c218da0e00 x1862023741930240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766130.311419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741930240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766130.311425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766130.311427:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766130.311428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930240:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766131.335066:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766132.359131:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766133.383050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766134.407160:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766135.431005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766135.431013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766135.431018:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775766135.431020:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766135.431027:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2023 00000100:00080000:1.0:1775766135.431040:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766135.431052:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766135.431087:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741930368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766135.431109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766135.431112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766135.431114:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766135.431117:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2023 00000100:00080000:1.0:1775766135.431119:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766135.431122:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766135.431135:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766135.431144:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741930496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766135.431154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766135.431154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741930368:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766135.431159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766135.431162:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766135.431167:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2023 00000100:00080000:1.0:1775766135.431169:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766135.431171:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766135.431174:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766135.431183:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741930624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766135.431191:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766135.431193:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766135.431195:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766135.431198:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2023 00000100:00080000:1.0:1775766135.431201:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766135.431203:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766135.431206:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766135.431215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741930752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766135.431220:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766135.431259:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766135/real 1775766135] req@ffff90c203496d80 x1862023741930368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766135.431273:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741930368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766135.431282:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766135.431286:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766135.431289:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741930368:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766135.431313:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930496:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766135.431337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766135.431369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766135/real 1775766135] req@ffff90c203495f80 x1862023741930496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766135.431385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741930496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766135.431393:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766135.431396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766135.431398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766135.431404:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766135.431452:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766135.431478:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766135/real 1775766135] req@ffff90c203494000 x1862023741930624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766135.431487:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741930624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766135.431493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766135.431495:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766135.431498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766135.431503:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930752:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766135.431516:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766135.431522:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766135/real 1775766135] req@ffff90c203497480 x1862023741930752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766135.431530:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741930752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766135.431538:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766135.431539:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766135.431541:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741930752:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766136.455061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766137.479063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766138.503070:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766139.527018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766140.550999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766140.551009:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766140.551016:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766140.551027:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2029 00000100:00080000:1.0:1775766140.551041:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766140.551049:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766140.551083:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766140.551096:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741930880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766140.551121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766140.551124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766140.551125:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766140.551128:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2029 00000100:00080000:1.0:1775766140.551131:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766140.551133:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766140.551144:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766140.551155:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741931008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766140.551164:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766140.551168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766140.551170:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766140.551170:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741930880:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766140.551182:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2029 00000100:00080000:1.0:1775766140.551188:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766140.551191:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766140.551194:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766140.551204:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741931136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766140.551211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766140.551214:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766140.551216:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766140.551220:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2029 00000100:00080000:1.0:1775766140.551226:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766140.551228:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766140.551231:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766140.551239:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741931264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766140.551248:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766140.551287:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766140/real 1775766140] req@ffff90c203494e00 x1862023741930880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766140.551302:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741930880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766140.551311:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766140.551314:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766140.551317:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741930880:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766140.551337:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766140.551352:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766140.551360:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766140/real 1775766140] req@ffff90c203497480 x1862023741931008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766140.551395:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741931008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766140.551401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766140.551404:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766140.551406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766140.551413:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766140.551440:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766140.551452:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766140/real 1775766140] req@ffff90c203494000 x1862023741931136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766140.551466:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741931136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766140.551474:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766140.551478:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766140.551481:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766140.551487:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931264:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766140.551505:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766140.551513:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766140/real 1775766140] req@ffff90c203495f80 x1862023741931264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766140.551520:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741931264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766140.551531:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766140.551534:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766140.551536:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931264:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766141.574995:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766142.599078:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766143.622974:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766144.647083:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766145.671029:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:1.0:1775766145.671039:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766145.671040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766145.671045:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766145.671057:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2034 00000100:00080000:3.0:1775766145.671069:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766145.671076:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766145.671122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741931392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766145.671147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766145.671150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766145.671152:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766145.671155:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2034 00000100:00080000:3.0:1775766145.671157:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766145.671161:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766145.671171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766145.671180:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741931520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766145.671192:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741931392:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766145.671196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766145.671207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766145.671209:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766145.671216:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2034 00000100:00080000:3.0:1775766145.671222:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766145.671227:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766145.671235:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766145.671252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741931648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766145.671264:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775766145.671274:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766145.671278:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766145.671280:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766145.671288:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2034 00000100:00080000:3.0:1775766145.671294:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766145.671299:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766145.671304:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766145.671336:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741931776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766145.671351:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766145/real 1775766145] req@ffff90c218da0a80 x1862023741931392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766145.671375:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741931392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766145.671384:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766145.671388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766145.671390:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741931392:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766145.671413:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931520:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766145.671432:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766145.671441:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766145/real 1775766145] req@ffff90c218da3480 x1862023741931520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766145.671449:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741931520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766145.671455:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766145.671457:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766145.671459:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766145.671464:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931648:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766145.671476:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766145.671481:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766145/real 1775766145] req@ffff90c218da0700 x1862023741931648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766145.671489:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741931648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766145.671493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766145.671495:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766145.671497:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766145.671501:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931776:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766145.671511:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766145.671516:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766145/real 1775766145] req@ffff90c218da1f80 x1862023741931776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766170 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766145.671522:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741931776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766170 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766145.671528:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766145.671530:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766145.671532:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741931776:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766146.695013:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766147.719105:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766148.743125:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766149.767083:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766150.790967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766150.790978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766150.790982:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766150.790995:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2039 00000100:00080000:3.0:1775766150.791007:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766150.791014:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766150.791040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766150.791059:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741931904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766150.791082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766150.791085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766150.791087:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766150.791089:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2039 00000100:00080000:3.0:1775766150.791092:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766150.791095:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766150.791105:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766150.791117:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741931904:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766150.791117:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741932032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766150.791132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766150.791134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766150.791135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766150.791137:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2039 00000100:00080000:3.0:1775766150.791139:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766150.791142:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766150.791145:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766150.791154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741932160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766150.791161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766150.791164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766150.791165:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766150.791168:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2039 00000100:00080000:3.0:1775766150.791189:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766150.791194:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766150.791199:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766150.791205:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766150.791215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741932288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766150.791258:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766150/real 1775766150] req@ffff90c218da0000 x1862023741931904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766150.791315:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741931904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766150.791328:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766150.791333:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766150.791338:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741931904:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766150.791366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766150.791390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766150.791402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766150/real 1775766150] req@ffff90c218da0380 x1862023741932032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766150.791415:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741932032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766150.791425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766150.791428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766150.791431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766150.791438:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766150.791457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766150.791467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766150/real 1775766150] req@ffff90c218da3100 x1862023741932160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766150.791480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741932160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766150.791496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766150.791500:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766150.791503:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766150.791510:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932288:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766150.791531:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766150.791540:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766150/real 1775766150] req@ffff90c218da1f80 x1862023741932288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766175 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766150.791551:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741932288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766175 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766150.791560:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766150.791564:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766150.791566:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932288:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766151.815134:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766152.839087:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766153.863064:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766154.887146:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766155.911026:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766155.911053:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766155.911062:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766155.911067:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766155.911079:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2044 00000100:00080000:0.0:1775766155.911092:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766155.911099:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766155.911146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741932416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766155.911189:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766155.911192:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766155.911194:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766155.911198:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2044 00000100:00080000:0.0:1775766155.911200:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766155.911203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766155.911214:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766155.911257:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741932544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766155.911285:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741932416:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766155.911287:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766155.911304:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766155.911306:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766155.911311:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2044 00000100:00080000:0.0:1775766155.911313:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766155.911317:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766155.911323:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766155.911354:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741932672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766155.911361:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766155.911364:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766155.911365:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766155.911368:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2044 00000100:00080000:0.0:1775766155.911374:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766155.911377:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766155.911383:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766155.911395:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741932800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766155.911396:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766155.911442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766155/real 1775766155] req@ffff90c20363aa00 x1862023741932416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766155.911461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741932416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766155.911473:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766155.911478:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766155.911493:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741932416:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766155.911519:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932544:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766155.911544:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766155.911556:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766155/real 1775766155] req@ffff90c203638000 x1862023741932544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766155.911565:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741932544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766155.911571:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766155.911573:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766155.911576:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766155.911580:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766155.911594:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766155.911600:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766155/real 1775766155] req@ffff90c203639180 x1862023741932672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766155.911607:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741932672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766155.911611:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766155.911613:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766155.911615:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766155.911619:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932800:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766155.911629:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766155.911634:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766155/real 1775766155] req@ffff90c203639500 x1862023741932800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766155.911641:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741932800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766180 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766155.911648:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766155.911650:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766155.911652:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741932800:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766156.935060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766157.959075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766158.983019:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766160.007178:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766161.031007:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766161.031015:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766161.031020:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766161.031030:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2049 00000100:00080000:1.0:1775766161.031040:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766161.031046:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766161.031077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766161.031084:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741932928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766161.031109:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766161.031112:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766161.031113:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766161.031116:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2049 00000100:00080000:1.0:1775766161.031119:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766161.031122:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766161.031130:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766161.031138:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741933056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766161.031148:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766161.031154:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766161.031156:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766161.031156:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741932928:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766161.031163:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2049 00000100:00080000:1.0:1775766161.031190:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766161.031196:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766161.031202:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766161.031212:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741933184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766161.031220:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766161.031224:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766161.031226:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766161.031229:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2049 00000100:00080000:1.0:1775766161.031231:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766161.031234:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766161.031237:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766161.031248:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741933312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766161.031301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766161.031342:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766161.031361:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933312:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766162.055009:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766163.079032:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766164.103117:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766164.103245:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775766164.103286:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766164.103308:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766164.103357:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766164.103368:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766164.103378:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766164.103789:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766161/real 1775766164] req@ffff90c203494380 x1862023741933056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766164.103821:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741933056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766164.103834:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766164.103840:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766164.103845:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766164.103892:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766161/real 1775766164] req@ffff90c203497100 x1862023741933184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766164.103910:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741933184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766164.103920:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766164.103925:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766164.103929:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766164.103937:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766161/real 1775766164] req@ffff90c203495880 x1862023741933312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766164.103951:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741933312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766164.103963:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766164.103967:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766164.103972:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933312:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766164.103980:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766161/real 1775766164] req@ffff90c203496d80 x1862023741932928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766164.103995:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741932928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766164.104005:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766164.104009:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766164.104012:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741932928:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766165.127082:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766166.151023:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766166.151034:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766166.151039:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:0.0:1775766166.151043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766166.151049:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2054 00000100:00080000:3.0:1775766166.151063:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766166.151082:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766166.151152:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741933440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766166.151193:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766166.151197:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766166.151199:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766166.151203:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2054 00000100:00080000:3.0:1775766166.151208:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766166.151214:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766166.151226:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766166.151247:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741933440:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766166.151247:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741933568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766166.151265:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766166.151269:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766166.151272:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766166.151288:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2054 00000100:00080000:3.0:1775766166.151292:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766166.151297:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766166.151303:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766166.151316:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741933696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766166.151326:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766166.151331:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1775766166.151333:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766166.151335:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766166.151343:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2054 00000100:00080000:3.0:1775766166.151347:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766166.151351:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766166.151357:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766166.151374:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741933824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766166.151386:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766166/real 1775766166] req@ffff90c218da0700 x1862023741933440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766166.151419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741933440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766166.151430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766166.151434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766166.151438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741933440:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766166.151463:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766166.151486:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766166.151497:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766166/real 1775766166] req@ffff90c218da3480 x1862023741933568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766166.151511:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741933568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766166.151520:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766166.151525:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766166.151528:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766166.151534:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766166.151557:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766166.151576:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766166/real 1775766166] req@ffff90c218da0a80 x1862023741933696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766166.151590:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741933696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766166.151606:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766166.151611:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766166.151614:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766166.151621:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933824:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766166.151641:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766166.151652:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766166/real 1775766166] req@ffff90c218da0e00 x1862023741933824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766166.151664:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741933824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766166.151675:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766166.151679:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766166.151683:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741933824:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766167.175077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766168.199043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766169.223120:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766170.247052:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766171.270984:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766171.270991:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766171.270995:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766171.271003:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2059 00000100:00080000:1.0:1775766171.271014:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766171.271020:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766171.271037:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766171.271062:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741933952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766171.271097:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766171.271099:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766171.271101:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766171.271104:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2059 00000100:00080000:1.0:1775766171.271107:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766171.271109:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766171.271118:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766171.271127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741934080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766171.271135:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766171.271140:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775766171.271140:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741933952:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766171.271142:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766171.271148:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2059 00000100:00080000:1.0:1775766171.271151:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766171.271153:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766171.271156:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766171.271164:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741934208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766171.271172:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766171.271174:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766171.271176:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766171.271179:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2059 00000100:00080000:1.0:1775766171.271182:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766171.271184:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766171.271187:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766171.271194:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741934336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766171.271204:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766171.271236:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766171/real 1775766171] req@ffff90c203495c00 x1862023741933952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766171.271251:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741933952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766171.271259:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766171.271262:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766171.271265:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741933952:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766171.271283:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934080:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766171.271296:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766171.271304:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766171/real 1775766171] req@ffff90c203497b80 x1862023741934080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766171.271312:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741934080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766171.271317:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766171.271320:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766171.271322:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766171.271325:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934208:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766171.271334:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766171.271339:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766171/real 1775766171] req@ffff90c203496300 x1862023741934208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766171.271346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741934208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766171.271351:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766171.271353:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766171.271364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766171.271371:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934336:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766171.271381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766171.271387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766171/real 1775766171] req@ffff90c203496a00 x1862023741934336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766171.271395:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741934336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766171.271402:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766171.271404:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766171.271406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934336:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766172.295023:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766173.319008:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766174.343006:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766175.367046:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766176.391077:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775766176.391087:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766176.391098:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766176.391108:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766176.391127:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2064 00000100:00080000:1.0:1775766176.391147:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766176.391159:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766176.391223:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741934464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766176.391273:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766176.391280:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766176.391284:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766176.391293:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2064 00000100:00080000:1.0:1775766176.391302:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766176.391310:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766176.391324:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741934464:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775766176.391327:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766176.391356:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741934592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766176.391388:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766176.391394:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766176.391396:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766176.391402:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2064 00000100:00080000:1.0:1775766176.391408:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766176.391418:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766176.391420:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775766176.391435:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766176.391455:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741934720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766176.391482:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766176/real 1775766176] req@ffff90c203496680 x1862023741934464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766176.391490:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766176.391504:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766176.391536:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766176.391546:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2064 00000100:00100000:2.0:1775766176.391547:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741934464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766176.391559:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766176.391574:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775766176.391575:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1775766176.391587:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775766176.391588:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766176.391600:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741934464:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766176.391607:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741934848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766176.391653:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934592:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766176.391682:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766176.391693:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766176/real 1775766176] req@ffff90c203494a80 x1862023741934592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766176.391702:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741934592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766176.391713:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766176.391716:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766176.391718:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766176.391723:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766176.391735:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766176.391743:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766176/real 1775766176] req@ffff90c203495180 x1862023741934720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766176.391755:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741934720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766176.391763:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766176.391767:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766176.391770:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766176.391777:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934848:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766176.391798:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766176.391809:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766176/real 1775766176] req@ffff90c203495500 x1862023741934848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766176.391817:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741934848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766176.391825:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766176.391827:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766176.391841:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741934848:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766177.415043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766178.439048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766179.463100:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766180.487034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766181.511010:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766181.511020:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766181.511025:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766181.511037:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2069 00000100:00080000:1.0:1775766181.511050:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766181.511056:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766181.511067:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766181.511093:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741934976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766181.511126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766181.511130:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766181.511132:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766181.511137:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2069 00000100:00080000:1.0:1775766181.511142:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766181.511147:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766181.511157:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766181.511160:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741934976:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766181.511172:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741935104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766181.511181:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766181.511184:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766181.511185:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766181.511188:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2069 00000100:00080000:1.0:1775766181.511191:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766181.511194:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766181.511197:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766181.511205:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741935232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766181.511211:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766181.511213:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766181.511214:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766181.511217:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2069 00000100:00080000:1.0:1775766181.511220:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766181.511222:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766181.511226:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766181.511231:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766181.511236:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741935360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766181.511274:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766181/real 1775766181] req@ffff90c203494700 x1862023741934976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766181.511289:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741934976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766181.511296:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766181.511299:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766181.511302:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741934976:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766181.511320:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766181.511336:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766181.511344:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766181/real 1775766181] req@ffff90c203495500 x1862023741935104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766181.511352:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741935104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766181.511357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766181.511359:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766181.511361:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766181.511365:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766181.511373:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766181.511379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766181/real 1775766181] req@ffff90c203495180 x1862023741935232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766181.511386:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741935232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766181.511391:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766181.511393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766181.511394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766181.511398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935360:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766181.511407:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766181.511412:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766181/real 1775766181] req@ffff90c203494a80 x1862023741935360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766181.511418:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741935360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766181.511424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766181.511426:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766181.511428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935360:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766182.535131:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766183.559089:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766184.583091:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766185.607063:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766186.631031:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766186.631039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766186.631044:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766186.631052:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2075 00000100:00080000:3.0:1775766186.631063:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766186.631068:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766186.631079:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766186.631105:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741935488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766186.631129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766186.631133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766186.631135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766186.631139:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2075 00000100:00080000:3.0:1775766186.631143:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766186.631148:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766186.631158:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766186.631163:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741935488:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766186.631175:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741935616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766186.631189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766186.631193:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766186.631195:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766186.631198:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2075 00000100:00080000:3.0:1775766186.631215:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766186.631218:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766186.631223:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766186.631225:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766186.631236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741935744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766186.631245:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766186.631247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766186.631248:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766186.631251:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2075 00000100:00080000:3.0:1775766186.631254:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766186.631256:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766186.631260:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766186.631265:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766186/real 1775766186] req@ffff90c218da1880 x1862023741935488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766186.631272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741935872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766186.631289:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741935488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766186.631299:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766186.631303:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766186.631305:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741935488:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766186.631324:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935616:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766186.631339:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766186.631346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766186/real 1775766186] req@ffff90c218da3800 x1862023741935616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766186.631354:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741935616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766186.631359:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766186.631361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766186.631363:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766186.631366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935744:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766186.631376:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766186.631382:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766186/real 1775766186] req@ffff90c218da2d80 x1862023741935744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766186.631389:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741935744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766186.631394:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766186.631395:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766186.631397:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766186.631401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935872:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766186.631411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766186.631416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766186/real 1775766186] req@ffff90c218da0e00 x1862023741935872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766211 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766186.631423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741935872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766211 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766186.631429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766186.631431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766186.631433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741935872:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766187.655108:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766188.679111:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766189.703038:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766190.727078:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766191.750972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766191.750981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766191.750986:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766191.750996:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2080 00000100:00080000:3.0:1775766191.751009:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766191.751017:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766191.751034:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766191.751060:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741936000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766191.751085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766191.751088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766191.751090:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766191.751093:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2080 00000100:00080000:3.0:1775766191.751097:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766191.751100:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766191.751110:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766191.751117:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741936000:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766191.751122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741936128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766191.751131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766191.751133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766191.751135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766191.751138:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2080 00000100:00080000:3.0:1775766191.751140:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766191.751142:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766191.751145:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766191.751154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741936256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766191.751160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766191.751162:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766191.751164:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766191.751167:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2080 00000100:00080000:3.0:1775766191.751171:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766191.751174:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766191.751177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766191.751181:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766191.751187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741936384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766191.751226:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766191/real 1775766191] req@ffff90c218da0a80 x1862023741936000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766191.751241:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741936000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766191.751249:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766191.751252:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766191.751255:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741936000:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766191.751273:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766191.751288:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766191.751296:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766191/real 1775766191] req@ffff90c218da3480 x1862023741936128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766191.751304:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741936128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766191.751310:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766191.751312:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766191.751314:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766191.751317:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766191.751327:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766191.751333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766191/real 1775766191] req@ffff90c218da0700 x1862023741936256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766191.751340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741936256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766191.751344:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766191.751346:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766191.751348:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766191.751351:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936384:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766191.751361:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766191.751366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766191/real 1775766191] req@ffff90c218da1f80 x1862023741936384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766216 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766191.751372:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741936384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766216 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766191.751384:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766191.751386:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766191.751388:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936384:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766192.775107:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766193.799033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766194.823120:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766195.847061:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766196.871002:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766196.871013:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766196.871019:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766196.871030:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2085 00010000:00010000:3.0:1775766196.871031:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766196.871044:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766196.871053:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766196.871088:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741936512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766196.871113:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766196.871115:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766196.871117:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766196.871120:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2085 00000100:00080000:0.0:1775766196.871123:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766196.871126:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766196.871134:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766196.871146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741936640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766196.871149:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741936512:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766196.871157:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766196.871160:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766196.871161:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766196.871164:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2085 00000100:00080000:0.0:1775766196.871167:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766196.871169:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766196.871172:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766196.871179:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741936768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766196.871185:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766196.871187:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766196.871188:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766196.871192:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2085 00000100:00080000:0.0:1775766196.871194:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766196.871196:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766196.871199:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766196.871206:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741936896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766196.871214:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766196.871248:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766196/real 1775766196] req@ffff90c203638700 x1862023741936512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766196.871262:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741936512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766196.871271:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766196.871274:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766196.871277:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741936512:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766196.871294:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766196.871308:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766196.871315:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766196/real 1775766196] req@ffff90c203639500 x1862023741936640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766196.871323:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741936640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766196.871329:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766196.871332:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766196.871334:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766196.871338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766196.871348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766196.871353:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766196/real 1775766196] req@ffff90c203639180 x1862023741936768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766196.871360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741936768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766196.871365:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766196.871366:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766196.871368:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766196.871372:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936896:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766196.871381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766196.871385:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766196/real 1775766196] req@ffff90c203638000 x1862023741936896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766196.871392:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741936896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766221 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766196.871398:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766196.871400:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766196.871402:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741936896:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766197.894986:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766198.919085:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766199.942993:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766200.967082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766201.990965:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766201.990977:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766201.990982:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766201.990993:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2090 00000100:00080000:0.0:1775766201.991005:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766201.991014:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766201.991034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766201.991057:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741937024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766201.991097:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766201.991101:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766201.991103:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766201.991108:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2090 00000100:00080000:0.0:1775766201.991113:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766201.991119:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766201.991129:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766201.991132:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741937024:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766201.991157:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741937152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766201.991170:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766201.991173:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766201.991175:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766201.991178:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2090 00000100:00080000:0.0:1775766201.991183:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766201.991186:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766201.991190:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766201.991201:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741937280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766201.991208:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766201.991215:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766201.991218:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766201.991219:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766201.991223:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2090 00000100:00080000:0.0:1775766201.991227:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766201.991230:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766201.991233:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766201.991241:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741937408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766201.991249:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766201/real 1775766201] req@ffff90c20363aa00 x1862023741937024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766201.991268:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741937024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766201.991277:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766201.991280:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766201.991283:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741937024:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766201.991301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937152:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766201.991326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766201.991334:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766201/real 1775766201] req@ffff90c20363b100 x1862023741937152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766201.991342:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741937152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766201.991348:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766201.991350:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766201.991353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766201.991356:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766201.991366:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766201.991372:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766201/real 1775766201] req@ffff90c20363bb80 x1862023741937280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766201.991379:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741937280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766201.991389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766201.991391:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766201.991393:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766201.991397:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937408:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766201.991407:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766201.991413:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766201/real 1775766201] req@ffff90c20363a680 x1862023741937408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766201.991420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741937408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766226 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766201.991426:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766201.991428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766201.991431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937408:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766203.015088:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766204.039033:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766205.063159:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766206.087229:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766207.111020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766207.111029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766207.111034:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766207.111043:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2095 00000100:00080000:3.0:1775766207.111053:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766207.111059:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766207.111091:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766207.111100:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741937536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766207.111137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766207.111141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766207.111143:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766207.111148:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2095 00000100:00080000:3.0:1775766207.111154:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766207.111159:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766207.111170:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766207.111173:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741937536:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766207.111187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741937664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766207.111197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766207.111200:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766207.111202:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766207.111205:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2095 00000100:00080000:3.0:1775766207.111208:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766207.111211:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766207.111215:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766207.111225:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741937792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766207.111232:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766207.111234:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766207.111235:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766207.111238:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2095 00000100:00080000:3.0:1775766207.111242:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766207.111246:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766207.111248:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766207.111253:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766207.111264:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741937920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766207.111299:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766207/real 1775766207] req@ffff90c218da3100 x1862023741937536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766207.111317:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741937536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766207.111324:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766207.111328:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766207.111331:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741937536:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766207.111352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766207.111368:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766207.111384:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766207/real 1775766207] req@ffff90c218da0380 x1862023741937664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766207.111394:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741937664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766207.111400:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766207.111403:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766207.111405:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766207.111410:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766207.111422:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766207.111428:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766207/real 1775766207] req@ffff90c218da0000 x1862023741937792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766207.111436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741937792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766207.111442:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766207.111444:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766207.111452:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766207.111457:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937920:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766207.111470:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766207.111476:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766207/real 1775766207] req@ffff90c218da1f80 x1862023741937920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766207.111485:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741937920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766207.111492:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766207.111494:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766207.111496:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741937920:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766208.135170:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766209.159114:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766210.183225:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766211.207208:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766212.230997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766212.231012:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766212.231017:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766212.231025:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2100 00000100:00080000:3.0:1775766212.231036:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766212.231042:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766212.231078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741938048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775766212.231088:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766212.231101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766212.231105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766212.231106:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766212.231110:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2100 00000100:00080000:3.0:1775766212.231114:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766212.231117:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766212.231124:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766212.231136:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741938176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766212.231144:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741938048:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766212.231153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766212.231157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766212.231159:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766212.231164:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2100 00000100:00080000:3.0:1775766212.231168:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766212.231172:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766212.231178:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766212.231191:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741938304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766212.231198:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766212.231201:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766212.231203:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766212.231205:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2100 00000100:00080000:3.0:1775766212.231209:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766212.231228:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766212.231232:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766212.231237:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766212.231247:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741938432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766212.231271:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766212/real 1775766212] req@ffff90c218da0700 x1862023741938048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766212.231287:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741938048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766212.231295:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766212.231298:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766212.231301:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741938048:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766212.231320:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938176:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766212.231341:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766212.231354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766212/real 1775766212] req@ffff90c218da3480 x1862023741938176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766212.231367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741938176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766212.231384:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766212.231388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766212.231392:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766212.231397:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938304:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766212.231413:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766212.231421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766212/real 1775766212] req@ffff90c218da0a80 x1862023741938304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766212.231429:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741938304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766212.231434:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766212.231436:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766212.231438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766212.231442:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938432:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766212.231452:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766212.231457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766212/real 1775766212] req@ffff90c218da0e00 x1862023741938432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766212.231465:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741938432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766212.231471:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766212.231473:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766212.231475:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938432:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766213.255042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766214.279036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766215.303075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766216.327010:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766217.350970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766217.350980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766217.350986:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766217.350996:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2105 00000100:00080000:2.0:1775766217.351008:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766217.351017:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766217.351057:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023741938560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766217.351116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766217.351122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766217.351125:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766217.351131:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2105 00000100:00080000:2.0:1775766217.351137:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766217.351143:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766217.351165:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766217.351183:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741938560:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766217.351186:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023741938688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766217.351202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766217.351206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766217.351207:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766217.351212:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2105 00000100:00080000:2.0:1775766217.351216:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766217.351221:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766217.351227:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766217.351237:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023741938816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766217.351246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766217.351250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766217.351251:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766217.351254:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2105 00000100:00080000:2.0:1775766217.351258:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775766217.351263:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775766217.351269:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766217.351277:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766217.351292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023741938944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766217.351309:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766217/real 1775766217] req@ffff90c2034bd180 x1862023741938560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766217.351338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023741938560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766217.351354:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766217.351361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766217.351369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741938560:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766217.351390:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766217.351405:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766217.351431:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766217.351442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766217/real 1775766217] req@ffff90c2034bdf80 x1862023741938688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766217.351453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023741938688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766217.351477:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766217.351480:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766217.351484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766217.351488:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766217.351502:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766217.351509:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766217/real 1775766217] req@ffff90c2034bf100 x1862023741938816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766217.351518:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023741938816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766217.351526:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766217.351528:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766217.351531:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766217.351535:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938944:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766217.351548:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766217.351554:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766217/real 1775766217] req@ffff90c2034bc700 x1862023741938944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766217.351563:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023741938944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766217.351572:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766217.351574:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766217.351578:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741938944:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766218.375155:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766219.399127:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766220.423060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766221.447046:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766222.470989:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766222.470997:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766222.471019:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766222.471033:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2110 00000100:00080000:1.0:1775766222.471050:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766222.471061:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766222.471097:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766222.471133:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741939072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766222.471191:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766222.471194:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766222.471202:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766222.471205:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2110 00000100:00080000:1.0:1775766222.471209:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766222.471212:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766222.471221:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766222.471232:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741939200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766222.471241:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766222.471245:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775766222.471247:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741939072:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766222.471247:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766222.471253:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2110 00000100:00080000:1.0:1775766222.471257:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766222.471260:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766222.471273:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766222.471280:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741939328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766222.471287:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766222.471289:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766222.471290:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766222.471292:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2110 00000100:00080000:1.0:1775766222.471295:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766222.471297:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766222.471300:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766222.471306:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741939456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766222.471335:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766222.471379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766222/real 1775766222] req@ffff90c203496680 x1862023741939072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766222.471401:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741939072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766222.471411:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766222.471415:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766222.471420:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741939072:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766222.471443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766222.471476:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766222.471487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766222/real 1775766222] req@ffff90c203496a00 x1862023741939200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766222.471501:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741939200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766222.471509:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766222.471513:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766222.471517:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766222.471523:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766222.471546:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766222.471557:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766222/real 1775766222] req@ffff90c203496300 x1862023741939328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766222.471570:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741939328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766222.471578:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766222.471582:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766222.471673:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766222.471680:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939456:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766222.471704:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766222.471715:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766222/real 1775766222] req@ffff90c203497b80 x1862023741939456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766222.471728:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741939456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766222.471738:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766222.471741:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766222.471745:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939456:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766223.495149:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766224.519128:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766225.543104:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766226.567052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766227.591012:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766227.591030:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766227.591036:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766227.591049:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2116 00000100:00080000:1.0:1775766227.591062:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775766227.591069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766227.591070:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766227.591120:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741939584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766227.591148:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766227.591152:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766227.591153:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766227.591156:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2116 00000100:00080000:1.0:1775766227.591159:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766227.591162:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766227.591171:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766227.591181:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741939712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766227.591191:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741939584:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766227.591198:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766227.591201:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766227.591203:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766227.591206:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2116 00000100:00080000:1.0:1775766227.591207:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766227.591209:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766227.591215:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766227.591224:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741939840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766227.591231:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766227.591234:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766227.591235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766227.591238:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2116 00000100:00080000:1.0:1775766227.591251:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766227.591254:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766227.591258:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766227.591268:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741939968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766227.591313:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766227.591341:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766227.591357:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939968:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766228.615208:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766229.639144:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766230.663147:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775766230.663183:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775766230.663221:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766230.663236:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766230.663267:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766230.663275:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766230.663284:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775766230.663298:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766227/real 1775766230] req@ffff90c203496d80 x1862023741939712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766230.663320:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741939712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766230.663330:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766230.663334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766230.663337:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766230.663350:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766227/real 1775766230] req@ffff90c203495880 x1862023741939840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766230.663365:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741939840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766230.663373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766230.663377:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766230.663380:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766230.663387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766227/real 1775766230] req@ffff90c203497100 x1862023741939968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766230.663399:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741939968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766230.663409:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766230.663413:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766230.663417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741939968:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775766230.663423:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766227/real 1775766230] req@ffff90c203495c00 x1862023741939584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766230.663437:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741939584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766230.663444:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766230.663447:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766230.663450:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741939584:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766231.687068:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766232.711006:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766232.711015:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766232.711020:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766232.711028:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2121 00000100:00080000:1.0:1775766232.711039:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766232.711045:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766232.711078:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741940096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766232.711103:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766232.711106:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766232.711107:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766232.711110:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2121 00000100:00080000:1.0:1775766232.711113:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766232.711115:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766232.711123:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766232.711134:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741940224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766232.711142:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766232.711145:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00010000:00010000:3.0:1775766232.711146:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766232.711147:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766232.711152:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2121 00000100:00080000:1.0:1775766232.711154:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766232.711157:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766232.711161:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766232.711169:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741940352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766232.711176:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766232.711177:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741940096:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766232.711179:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766232.711190:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766232.711197:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2121 00000100:00080000:1.0:1775766232.711200:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766232.711202:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766232.711206:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766232.711212:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741940480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766232.711264:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766232.711304:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766232/real 1775766232] req@ffff90c203494380 x1862023741940096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766232.711326:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741940096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766232.711336:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766232.711339:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766232.711342:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741940096:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766232.711364:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766232.711386:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766232.711396:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766232/real 1775766232] req@ffff90c203495f80 x1862023741940224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766232.711425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741940224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766232.711433:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766232.711437:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766232.711440:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766232.711481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766232.711530:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766232.711547:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766232/real 1775766232] req@ffff90c203494000 x1862023741940352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766232.711565:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741940352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766232.711576:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766232.711582:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766232.711587:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766232.711598:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940480:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766232.711627:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766232.711642:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766232/real 1775766232] req@ffff90c203497480 x1862023741940480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766257 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766232.711669:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741940480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766257 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766232.711684:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766232.711689:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766232.711693:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940480:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766233.735182:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766234.759035:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766235.783080:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766236.807054:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766237.830958:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766237.830966:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766237.830970:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766237.830977:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2126 00000100:00080000:1.0:1775766237.830988:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766237.831005:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766237.831041:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741940608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766237.831069:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766237.831072:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766237.831074:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766237.831076:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2126 00000100:00080000:1.0:1775766237.831079:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766237.831081:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766237.831089:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766237.831098:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741940736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766237.831105:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766237.831107:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766237.831109:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766237.831121:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2126 00000100:00080000:1.0:1775766237.831123:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766237.831126:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766237.831130:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766237.831136:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741940864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766237.831143:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766237.831145:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766237.831146:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766237.831152:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2126 00000100:00080000:1.0:1775766237.831154:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775766237.831155:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741940608:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766237.831158:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766237.831165:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766237.831172:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741940992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766237.831219:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766237.831254:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766237/real 1775766237] req@ffff90c203494e00 x1862023741940608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766237.831270:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741940608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766237.831285:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766237.831288:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766237.831291:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741940608:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766237.831297:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940736:192.168.202.155@tcp:38:kworker.0 00010000:00010000:3.0:1775766237.831313:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1775766237.831321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766237.831331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766237/real 1775766237] req@ffff90c203497480 x1862023741940736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766237.831350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741940736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766237.831356:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766237.831358:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766237.831360:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766237.831378:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940864:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766237.831389:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766237.831394:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766237/real 1775766237] req@ffff90c203494000 x1862023741940864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766237.831407:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741940864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766237.831412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766237.831414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766237.831416:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766237.831419:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940992:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766237.831435:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766237.831460:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766237/real 1775766237] req@ffff90c203495f80 x1862023741940992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766237.831472:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741940992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766262 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766237.831483:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766237.831487:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766237.831491:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741940992:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766238.855038:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766239.879059:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766240.903021:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766241.927048:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766242.951003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766242.951011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766242.951021:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766242.951029:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2131 00000100:00080000:2.0:1775766242.951041:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766242.951047:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766242.951088:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bed80 x1862023741941120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766242.951117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766242.951121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766242.951123:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766242.951127:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2131 00000100:00080000:2.0:1775766242.951132:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766242.951136:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766242.951146:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766242.951150:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741941120:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766242.951162:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be680 x1862023741941248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766242.951174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766242.951177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766242.951179:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766242.951183:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2131 00000100:00080000:2.0:1775766242.951188:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766242.951192:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766242.951197:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766242.951211:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766242.951212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023741941376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766242.951223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766242.951226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766242.951228:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766242.951233:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2131 00000100:00080000:2.0:1775766242.951237:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766242.951241:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766242.951246:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766242.951248:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766242/real 1775766242] req@ffff90c2034bed80 x1862023741941120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766242.951261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023741941504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766242.951279:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bed80 x1862023741941120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766242.951288:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766242.951291:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766242.951294:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741941120:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766242.951327:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766242.951345:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766242.951356:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766242/real 1775766242] req@ffff90c2034be680 x1862023741941248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766242.951365:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be680 x1862023741941248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766242.951372:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766242.951374:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766242.951378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941248:192.168.202.155@tcp:38:kworker.0 00010000:00010000:3.0:1775766242.951379:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766242.951383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766242.951437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766242.951447:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766242/real 1775766242] req@ffff90c2034bc700 x1862023741941376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766242.951456:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023741941376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766242.951461:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766242.951463:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766242.951465:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766242.951469:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941504:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766242.951481:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766242.951497:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766242/real 1775766242] req@ffff90c2034bf100 x1862023741941504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766242.951504:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023741941504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766267 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766242.951511:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766242.951513:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766242.951514:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941504:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766243.975070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766244.999117:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766246.023117:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766247.047090:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766248.071035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766248.071046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766248.071051:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766248.071061:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2136 00000100:00080000:1.0:1775766248.071074:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766248.071082:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766248.071120:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741941632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766248.071131:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766248.071150:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766248.071154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766248.071155:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766248.071158:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2136 00000100:00080000:1.0:1775766248.071162:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766248.071165:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766248.071173:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766248.071181:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741941760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766248.071188:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766248.071190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766248.071192:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766248.071194:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2136 00000100:00080000:1.0:1775766248.071196:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766248.071198:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766248.071200:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766248.071208:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741941888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766248.071215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766248.071218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766248.071219:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766248.071222:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2136 00000100:00080000:1.0:1775766248.071224:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766248.071226:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766248.071230:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766248.071236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741942016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766248.071241:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741941632:192.168.202.155@tcp:250:kworker.0 00000800:00000100:0.0:1775766248.071315:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766248.071397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766248/real 1775766248] req@ffff90c203494380 x1862023741941632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766248.071426:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741941632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766248.071435:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766248.071440:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766248.071444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741941632:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766248.071450:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766248.071473:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766248.071491:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766248/real 1775766248] req@ffff90c203495c00 x1862023741941760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766248.071499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741941760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766248.071505:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766248.071507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766248.071509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766248.071512:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766248.071523:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766248.071528:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766248/real 1775766248] req@ffff90c203497100 x1862023741941888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766248.071535:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741941888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766248.071556:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766248.071558:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766248.071560:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741941888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766248.071578:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942016:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766248.071588:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766248.071594:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766248/real 1775766248] req@ffff90c203495880 x1862023741942016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766248.071601:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741942016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766248.071608:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766248.071610:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766248.071611:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942016:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766249.095261:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766250.119007:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766251.143122:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766252.167004:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766253.191026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766253.191039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766253.191046:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766253.191059:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2141 00000100:00080000:1.0:1775766253.191075:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766253.191084:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766253.191111:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766253.191136:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741942144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766253.191167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766253.191171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766253.191173:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766253.191178:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2141 00000100:00080000:1.0:1775766253.191182:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766253.191187:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766253.191199:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766253.191201:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741942144:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766253.191218:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741942272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766253.191231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766253.191236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766253.191238:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766253.191242:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2141 00000100:00080000:1.0:1775766253.191245:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766253.191249:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766253.191255:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766253.191265:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741942400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766253.191273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766253.191276:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1775766253.191279:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766253.191280:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766253.191286:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2141 00000100:00080000:1.0:1775766253.191289:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766253.191308:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766253.191313:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766253.191324:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741942528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766253.191345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766253/real 1775766253] req@ffff90c203496d80 x1862023741942144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766253.191362:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741942144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766253.191370:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766253.191373:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766253.191377:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741942144:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766253.191401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942272:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766253.191417:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766253.191424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766253/real 1775766253] req@ffff90c203497b80 x1862023741942272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766253.191433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741942272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766253.191439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766253.191442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766253.191444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766253.191448:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942400:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766253.191457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766253.191462:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766253/real 1775766253] req@ffff90c203496300 x1862023741942400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766253.191470:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741942400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766253.191475:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766253.191477:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766253.191479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766253.191482:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942528:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766253.191492:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766253.191497:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766253/real 1775766253] req@ffff90c203496a00 x1862023741942528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766253.191505:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741942528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766253.191512:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766253.191513:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766253.191515:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942528:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766254.215035:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766255.239162:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766256.263070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766257.287070:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766258.310994:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766258.311045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766258.311051:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766258.311056:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766258.311063:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2146 00000100:00080000:3.0:1775766258.311074:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766258.311080:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766258.311116:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741942656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766258.311133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766258.311136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766258.311138:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766258.311140:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2146 00000100:00080000:3.0:1775766258.311143:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766258.311147:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766258.311148:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741942656:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775766258.311154:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766258.311165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741942784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766258.311177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766258.311181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766258.311184:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766258.311188:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2146 00000100:00080000:3.0:1775766258.311192:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766258.311196:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766258.311202:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775766258.311208:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766258.311227:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741942912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766258.311237:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766258.311240:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766258.311257:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766258.311261:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2146 00000100:00080000:3.0:1775766258.311264:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766258.311267:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766258.311271:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766258.311284:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741943040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766258.311296:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766258/real 1775766258] req@ffff90c218da2d80 x1862023741942656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766258.311321:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741942656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766258.311332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766258.311337:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766258.311340:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741942656:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766258.311359:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766258.311376:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766258.311385:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766258/real 1775766258] req@ffff90c218da3800 x1862023741942784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766258.311393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741942784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766258.311403:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766258.311405:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766258.311407:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766258.311410:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766258.311420:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766258.311425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766258/real 1775766258] req@ffff90c218da1880 x1862023741942912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766258.311432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741942912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766258.311437:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766258.311438:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766258.311448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741942912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766258.311452:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943040:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766258.311462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766258.311468:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766258/real 1775766258] req@ffff90c218da0e00 x1862023741943040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766258.311475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741943040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766258.311481:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766258.311482:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766258.311485:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943040:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766259.335066:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766260.359215:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766261.383098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766262.407144:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766263.431022:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775766263.431025:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766263.431033:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766263.431039:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766263.431049:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2151 00000100:00080000:0.0:1775766263.431061:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766263.431067:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766263.431103:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741943168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766263.431134:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766263.431138:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766263.431140:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766263.431146:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2151 00000100:00080000:0.0:1775766263.431151:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766263.431156:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766263.431166:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766263.431167:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741943168:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766263.431187:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741943296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766263.431230:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766263.431235:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766263.431254:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766263.431259:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2151 00000100:00080000:0.0:1775766263.431265:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766263.431271:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766263.431276:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:0.0:1775766263.431281:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766263.431296:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741943424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766263.431304:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766263.431307:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766263.431309:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766263.431312:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2151 00000100:00080000:0.0:1775766263.431317:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766263.431317:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766263/real 1775766263] req@ffff90c203638380 x1862023741943168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766263.431325:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766263.431334:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766263.431347:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741943552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766263.431349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741943168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766263.431378:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766263.431384:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766263.431388:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741943168:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766263.431410:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766263.431427:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766263.431436:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766263/real 1775766263] req@ffff90c203639f80 x1862023741943296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766263.431445:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741943296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766263.431452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766263.431455:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766263.431459:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766263.431466:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943424:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766263.431509:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766263.431521:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766263/real 1775766263] req@ffff90c203638e00 x1862023741943424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766263.431534:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741943424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766263.431543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766263.431547:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766263.431550:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766263.431557:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943552:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766263.431579:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766263.431591:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766263/real 1775766263] req@ffff90c20363b480 x1862023741943552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766263.431604:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741943552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766263.431615:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766263.431618:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766263.431622:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943552:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766264.455085:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766265.479116:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766266.503061:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766267.527043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766268.551076:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766268.551091:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766268.551099:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:1.0:1775766268.551101:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766268.551118:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2157 00000100:00080000:3.0:1775766268.551136:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766268.551184:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766268.551236:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741943680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766268.551290:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766268.551295:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766268.551298:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766268.551303:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2157 00000100:00080000:3.0:1775766268.551308:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766268.551314:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766268.551327:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766268.551348:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741943680:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766268.551348:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741943808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766268.551374:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766268.551379:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766268.551381:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766268.551387:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2157 00000100:00080000:3.0:1775766268.551391:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766268.551395:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766268.551401:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766268.551434:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741943936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766268.551446:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766268.551451:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766268.551453:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766268.551457:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2157 00000100:00080000:3.0:1775766268.551462:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766268.551469:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766268.551475:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766268.551482:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766268.551499:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741944064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766268.551532:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766268/real 1775766268] req@ffff90c218da0a80 x1862023741943680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766268.551578:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741943680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766268.551591:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766268.551596:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766268.551601:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741943680:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766268.551628:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943808:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766268.551653:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766268.551666:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766268/real 1775766268] req@ffff90c218da3480 x1862023741943808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766268.551679:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741943808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766268.551688:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766268.551692:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766268.551695:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766268.551701:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943936:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766268.551716:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766268.551725:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766268/real 1775766268] req@ffff90c218da0700 x1862023741943936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766268.551737:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741943936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766268.551752:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766268.551755:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766268.551759:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741943936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766268.551764:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944064:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766268.551779:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766268.551786:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766268/real 1775766268] req@ffff90c218da1f80 x1862023741944064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766268.551797:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741944064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766268.551806:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766268.551810:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766268.551813:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944064:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766269.575191:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766270.599177:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766271.623190:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766272.647070:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766273.670979:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766273.671003:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766273.671009:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766273.671025:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2162 00000100:00080000:3.0:1775766273.671036:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766273.671044:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766273.671061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766273.671118:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741944192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766273.671147:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766273.671150:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766273.671152:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766273.671157:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2162 00000100:00080000:3.0:1775766273.671162:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766273.671167:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766273.671178:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741944192:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775766273.671184:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766273.671201:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741944320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766273.671226:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766273.671230:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766273.671232:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766273.671236:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2162 00000800:00000100:2.0:1775766273.671239:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766273.671242:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766273.671248:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766273.671254:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766273.671266:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741944448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766273.671277:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766273/real 1775766273] req@ffff90c218da0000 x1862023741944192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766273.671277:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766273.671287:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766273.671290:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766273.671297:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2162 00000100:00100000:2.0:1775766273.671301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741944192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766273.671301:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766273.671310:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775766273.671317:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775766273.671319:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775766273.671323:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766273.671327:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741944192:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766273.671335:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741944576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766273.671351:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766273.671369:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766273.671377:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766273/real 1775766273] req@ffff90c218da0380 x1862023741944320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766273.671385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741944320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766273.671390:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766273.671392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766273.671395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766273.671398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766273.671407:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766273.671413:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766273/real 1775766273] req@ffff90c218da3100 x1862023741944448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766273.671419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741944448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766273.671424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766273.671426:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766273.671428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766273.671431:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944576:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766273.671440:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766273.671445:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766273/real 1775766273] req@ffff90c218da1f80 x1862023741944576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766298 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766273.671452:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741944576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766298 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766273.671459:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766273.671460:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766273.671462:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944576:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766274.695107:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766275.719025:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766276.743144:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766277.767099:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766278.790965:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766278.790976:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766278.790982:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766278.790992:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2167 00000100:00080000:3.0:1775766278.791004:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766278.791019:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766278.791066:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766278.791076:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741944704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766278.791117:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766278.791120:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766278.791122:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766278.791126:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2167 00000100:00080000:3.0:1775766278.791130:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766278.791134:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766278.791144:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766278.791159:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741944832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766278.791161:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741944704:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766278.791176:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766278.791179:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766278.791181:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766278.791186:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2167 00000100:00080000:3.0:1775766278.791189:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766278.791193:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766278.791198:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766278.791208:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741944960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766278.791218:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766278.791221:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766278.791223:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766278.791227:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2167 00000100:00080000:3.0:1775766278.791230:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766278.791234:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766278.791238:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775766278.791243:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766278.791257:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741945088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766278.791276:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766278/real 1775766278] req@ffff90c218da0700 x1862023741944704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766278.791292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741944704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766278.791299:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766278.791302:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766278.791304:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741944704:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766278.791322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944832:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766278.791337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766278.791344:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766278/real 1775766278] req@ffff90c218da3480 x1862023741944832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766278.791351:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741944832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766278.791361:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766278.791363:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766278.791365:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766278.791368:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944960:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766278.791377:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766278.791382:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766278/real 1775766278] req@ffff90c218da0a80 x1862023741944960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766278.791389:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741944960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766278.791394:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766278.791396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766278.791397:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741944960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766278.791401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945088:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766278.791409:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766278.791414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766278/real 1775766278] req@ffff90c218da0e00 x1862023741945088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766303 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766278.791420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741945088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766303 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766278.791426:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766278.791428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766278.791430:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945088:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766279.815133:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766280.839074:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766281.863160:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766282.887132:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766283.911014:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766283.911022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766283.911027:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766283.911035:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2172 00000100:00080000:3.0:1775766283.911045:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766283.911052:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766283.911055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766283.911089:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741945216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766283.911117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766283.911120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766283.911122:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766283.911127:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2172 00000100:00080000:3.0:1775766283.911131:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766283.911136:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766283.911147:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766283.911148:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741945216:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766283.911165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741945344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766283.911178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766283.911181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766283.911183:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766283.911185:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2172 00000100:00080000:3.0:1775766283.911201:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766283.911204:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766283.911208:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766283.911211:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766283.911220:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741945472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766283.911227:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766283.911230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766283.911231:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766283.911234:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2172 00000100:00080000:3.0:1775766283.911237:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766283.911239:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766283.911242:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766283.911250:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741945600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766283.911278:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766283/real 1775766283] req@ffff90c218da1880 x1862023741945216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766283.911301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741945216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766283.911311:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766283.911314:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766283.911319:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741945216:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766283.911339:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766283.911357:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766283.911365:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766283/real 1775766283] req@ffff90c218da3800 x1862023741945344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766283.911374:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741945344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766283.911380:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766283.911382:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766283.911385:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766283.911388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766283.911407:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766283.911415:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766283/real 1775766283] req@ffff90c218da2d80 x1862023741945472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766283.911423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741945472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766283.911428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766283.911430:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766283.911433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766283.911436:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945600:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766283.911446:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766283.911451:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766283/real 1775766283] req@ffff90c218da0e00 x1862023741945600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766283.911459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741945600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766308 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766283.911465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766283.911467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766283.911469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945600:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766284.935043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766285.959090:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766286.983094:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766288.007061:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766289.030946:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766289.030968:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766289.030972:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766289.030996:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2177 00000100:00080000:3.0:1775766289.031006:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766289.031013:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766289.031046:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741945728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775766289.031057:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766289.031070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766289.031073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766289.031074:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766289.031077:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2177 00000100:00080000:3.0:1775766289.031080:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766289.031083:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766289.031090:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766289.031099:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741945856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766289.031104:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741945728:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766289.031124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766289.031127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766289.031128:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766289.031132:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2177 00000100:00080000:3.0:1775766289.031134:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766289.031136:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766289.031139:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766289.031147:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741945984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766289.031153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766289.031155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766289.031157:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766289.031159:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2177 00000100:00080000:3.0:1775766289.031162:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766289.031165:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766289.031167:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766289.031176:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766289.031177:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741946112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766289.031215:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766289/real 1775766289] req@ffff90c218da0a80 x1862023741945728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766289.031230:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741945728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766289.031239:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766289.031242:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766289.031245:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741945728:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766289.031262:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766289.031278:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766289.031286:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766289/real 1775766289] req@ffff90c218da3480 x1862023741945856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766289.031294:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741945856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766289.031299:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766289.031302:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766289.031304:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766289.031308:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766289.031318:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766289.031323:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766289/real 1775766289] req@ffff90c218da0700 x1862023741945984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766289.031331:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741945984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766289.031336:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766289.031338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766289.031340:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741945984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766289.031344:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946112:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766289.031354:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766289.031360:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766289/real 1775766289] req@ffff90c218da1f80 x1862023741946112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766289.031367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741946112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766289.031374:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766289.031376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766289.031377:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946112:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766290.055168:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766291.079027:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766292.103088:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766293.127052:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766294.150974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766294.150982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766294.150986:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766294.150994:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2182 00000100:00080000:3.0:1775766294.151005:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766294.151011:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766294.151041:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766294.151047:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741946240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766294.151076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766294.151079:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766294.151080:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766294.151083:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2182 00000100:00080000:3.0:1775766294.151085:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766294.151089:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766294.151097:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766294.151109:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741946368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766294.151110:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741946240:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766294.151123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766294.151126:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766294.151127:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766294.151130:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2182 00000100:00080000:3.0:1775766294.151133:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766294.151135:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766294.151138:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766294.151146:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741946496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766294.151153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766294.151156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766294.151157:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766294.151160:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2182 00000100:00080000:3.0:1775766294.151165:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766294.151167:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766294.151170:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766294.151178:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741946624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766294.151209:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766294.151235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766294.151251:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946624:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766295.175009:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766296.199160:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766297.223078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775766297.223171:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775766297.223221:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775766297.223257:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766297.223293:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766297.223304:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766297.223313:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766297.223344:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766294/real 1775766297] req@ffff90c218da0380 x1862023741946368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766297.223375:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741946368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766297.223389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766297.223394:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766297.223400:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766297.223409:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766294/real 1775766297] req@ffff90c218da0000 x1862023741946496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766297.223422:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741946496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766297.223430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766297.223433:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766297.223437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766297.223444:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766294/real 1775766297] req@ffff90c218da1f80 x1862023741946624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766297.223456:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741946624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766297.223467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766297.223470:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766297.223474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946624:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766297.223480:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766294/real 1775766297] req@ffff90c218da3100 x1862023741946240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766297.223502:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741946240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766297.223509:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766297.223513:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766297.223516:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741946240:192.168.202.155@tcp:250:kworker.0 00010000:00010000:0.0:1775766298.247033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766299.271075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766299.271076:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766299.271087:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766299.271093:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766299.271103:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2187 00000100:00080000:2.0:1775766299.271116:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766299.271123:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766299.271161:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023741946752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766299.271194:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766299.271198:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766299.271200:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766299.271204:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2187 00000100:00080000:2.0:1775766299.271208:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766299.271212:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766299.271222:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766299.271236:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023741946880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766299.271248:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766299.271249:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741946752:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775766299.271255:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766299.271259:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766299.271265:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2187 00000100:00080000:2.0:1775766299.271269:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766299.271273:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766299.271280:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766299.271293:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023741947008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766299.271303:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766299.271307:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766299.271308:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766299.271313:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2187 00000100:00080000:2.0:1775766299.271318:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766299.271325:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:0.0:1775766299.271327:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:2.0:1775766299.271333:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766299.271347:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023741947136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766299.271376:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766299/real 1775766299] req@ffff90c2034bdf80 x1862023741946752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766299.271401:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023741946752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766299.271412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766299.271417:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766299.271421:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741946752:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766299.271449:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766299.271470:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766299.271481:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766299/real 1775766299] req@ffff90c2034bd180 x1862023741946880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766299.271494:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023741946880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766299.271503:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766299.271507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766299.271511:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741946880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766299.271517:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766299.271543:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766299.271555:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766299/real 1775766299] req@ffff90c2034bc380 x1862023741947008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766299.271568:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023741947008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766299.271577:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766299.271580:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766299.271583:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766299.271590:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947136:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766299.271611:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766299.271622:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766299/real 1775766299] req@ffff90c2034bce00 x1862023741947136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766299.271634:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023741947136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766299.271646:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766299.271650:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766299.271653:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947136:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766300.295049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766301.319145:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766302.343124:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766303.367100:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766304.390985:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766304.390996:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766304.391001:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766304.391011:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2192 00000100:00080000:2.0:1775766304.391025:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766304.391031:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766304.391067:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023741947264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766304.391094:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766304.391097:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766304.391098:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766304.391101:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2192 00000100:00080000:2.0:1775766304.391105:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766304.391107:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766304.391114:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766304.391124:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023741947392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766304.391131:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766304.391133:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766304.391134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766304.391137:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2192 00000100:00080000:2.0:1775766304.391139:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766304.391142:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766304.391144:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766304.391151:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023741947520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766304.391158:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766304.391163:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766304.391167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766304.391171:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741947264:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775766304.391172:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2192 00000100:00080000:2.0:1775766304.391179:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766304.391187:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766304.391190:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766304.391203:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023741947648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766304.391260:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00010000:00010000:3.0:1775766304.391303:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766304.391305:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766304/real 1775766304] req@ffff90c2034bca80 x1862023741947264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766304.391327:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023741947264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766304.391337:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766304.391341:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766304.391345:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741947264:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766304.391351:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947392:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766304.391367:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766304.391379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766304/real 1775766304] req@ffff90c2034bea00 x1862023741947392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766304.391392:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023741947392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766304.391401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766304.391405:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766304.391409:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766304.391446:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947520:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766304.391464:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766304.391471:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766304/real 1775766304] req@ffff90c2034bf480 x1862023741947520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766304.391480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023741947520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766304.391496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766304.391500:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766304.391503:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766304.391510:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947648:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766304.391541:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766304.391550:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766304/real 1775766304] req@ffff90c2034be300 x1862023741947648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766304.391559:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023741947648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766304.391567:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766304.391570:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766304.391572:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947648:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766305.415015:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766306.438992:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766307.463048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766308.487010:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766309.511100:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766309.511112:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766309.511118:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766309.511142:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2197 00010000:00010000:1.0:1775766309.511154:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766309.511155:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766309.511165:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766309.511225:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741947776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766309.511256:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766309.511259:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766309.511261:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766309.511264:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2197 00000100:00080000:3.0:1775766309.511268:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766309.511271:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766309.511281:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766309.511291:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741947904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766309.511301:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766309.511304:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766309.511307:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766309.511315:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741947776:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766309.511316:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2197 00000100:00080000:3.0:1775766309.511320:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766309.511330:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766309.511335:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766309.511345:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741948032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766309.511353:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766309.511356:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766309.511358:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766309.511362:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2197 00000100:00080000:3.0:1775766309.511365:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766309.511368:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766309.511372:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766309.511381:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741948160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766309.511406:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766309.511444:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766309/real 1775766309] req@ffff90c218da0700 x1862023741947776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766309.511460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741947776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766309.511468:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766309.511472:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766309.511476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741947776:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766309.511497:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947904:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766309.511513:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766309.511521:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766309/real 1775766309] req@ffff90c218da3480 x1862023741947904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766309.511537:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741947904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766309.511543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766309.511546:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766309.511548:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741947904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766309.511554:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766309.511566:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766309.511572:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766309/real 1775766309] req@ffff90c218da0a80 x1862023741948032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766309.511580:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741948032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766309.511586:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766309.511589:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766309.511591:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766309.511595:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948160:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766309.511605:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766309.511611:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766309/real 1775766309] req@ffff90c218da0e00 x1862023741948160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766309.511619:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741948160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766309.511625:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766309.511628:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766309.511630:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948160:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766310.535032:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766311.559090:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766312.583055:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766313.607116:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766314.630981:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766314.630992:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766314.630997:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766314.631009:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2203 00000100:00080000:3.0:1775766314.631021:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766314.631029:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766314.631092:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766314.631093:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741948288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766314.631119:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766314.631122:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766314.631124:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766314.631127:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2203 00000100:00080000:3.0:1775766314.631130:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766314.631133:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766314.631141:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766314.631151:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741948416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766314.631161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766314.631164:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741948288:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766314.631165:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766314.631168:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766314.631175:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2203 00000100:00080000:3.0:1775766314.631179:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766314.631183:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766314.631187:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766314.631196:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741948544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766314.631203:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766314.631205:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766314.631207:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766314.631210:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2203 00000100:00080000:3.0:1775766314.631228:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766314.631231:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766314.631234:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766314.631239:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766314.631245:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741948672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766314.631281:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766314/real 1775766314] req@ffff90c218da2d80 x1862023741948288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766314.631296:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741948288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766314.631305:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766314.631308:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766314.631311:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741948288:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766314.631333:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766314.631347:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766314.631355:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766314/real 1775766314] req@ffff90c218da3800 x1862023741948416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766314.631363:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741948416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766314.631369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766314.631371:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766314.631374:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948416:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766314.631377:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948544:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766314.631386:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766314.631393:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766314/real 1775766314] req@ffff90c218da1880 x1862023741948544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766314.631400:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741948544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766314.631405:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766314.631406:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766314.631408:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766314.631411:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948672:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766314.631421:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766314.631426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766314/real 1775766314] req@ffff90c218da0e00 x1862023741948672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766339 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766314.631432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741948672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766339 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766314.631439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766314.631442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766314.631444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948672:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766315.655060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766316.679206:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766317.703152:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766318.727151:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766319.750980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766319.750988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766319.750993:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766319.751002:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2208 00000100:00080000:0.0:1775766319.751013:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766319.751020:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766319.751052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766319.751059:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741948800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766319.751091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766319.751094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766319.751096:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766319.751099:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2208 00000100:00080000:0.0:1775766319.751103:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766319.751107:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766319.751118:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766319.751132:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741948928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766319.751139:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741948800:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766319.751146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766319.751149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766319.751151:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766319.751155:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2208 00000100:00080000:0.0:1775766319.751157:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766319.751159:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766319.751162:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766319.751173:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741949056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766319.751178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766319.751181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766319.751182:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766319.751185:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2208 00000100:00080000:0.0:1775766319.751188:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766319.751190:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766319.751195:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766319.751200:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741949184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766319.751233:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766319.751278:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766319/real 1775766319] req@ffff90c20363b800 x1862023741948800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766319.751294:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741948800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766319.751305:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766319.751309:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766319.751313:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741948800:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766319.751337:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948928:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766319.751353:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766319.751362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766319/real 1775766319] req@ffff90c20363b480 x1862023741948928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766319.751371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741948928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766319.751378:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766319.751380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766319.751383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741948928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766319.751402:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949056:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766319.751418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766319.751426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766319/real 1775766319] req@ffff90c203638e00 x1862023741949056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766319.751436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741949056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766319.751443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766319.751446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766319.751448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766319.751455:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949184:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766319.751469:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766319.751476:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766319/real 1775766319] req@ffff90c203639f80 x1862023741949184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766344 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766319.751486:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741949184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766344 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766319.751495:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766319.751498:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766319.751500:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949184:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766320.775101:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766321.799166:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766322.823197:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766323.847146:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766324.871073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766324.871088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766324.871095:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:1.0:1775766324.871106:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766324.871112:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2213 00000100:00080000:3.0:1775766324.871129:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766324.871140:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766324.871193:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741949312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766324.871239:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766324.871246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766324.871249:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766324.871257:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2213 00000100:00080000:3.0:1775766324.871273:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766324.871280:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766324.871291:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741949312:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775766324.871295:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766324.871323:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741949440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766324.871355:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766324.871362:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766324.871365:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766324.871371:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2213 00000100:00080000:3.0:1775766324.871379:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766324.871384:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766324.871392:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775766324.871397:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766324.871421:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741949568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766324.871438:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766324.871442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766324/real 1775766324] req@ffff90c218da0a80 x1862023741949312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766324.871449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766324.871455:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766324.871469:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2213 00000100:00080000:3.0:1775766324.871476:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766324.871477:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741949312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766324.871491:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775766324.871499:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775766324.871504:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775766324.871508:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766324.871516:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741949312:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766324.871534:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741949696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766324.871546:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766324.871584:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766324.871600:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766324/real 1775766324] req@ffff90c218da3480 x1862023741949440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766324.871639:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741949440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766324.871660:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766324.871663:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766324.871666:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766324.871673:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766324.871691:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766324.871701:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766324/real 1775766324] req@ffff90c218da0700 x1862023741949568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766324.871713:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741949568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766324.871720:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766324.871723:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766324.871726:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766324.871741:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949696:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766324.871757:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766324.871765:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766324/real 1775766324] req@ffff90c218da3100 x1862023741949696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766324.871776:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741949696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766349 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766324.871793:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766324.871796:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766324.871799:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949696:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766325.895034:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766326.919001:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766327.943058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766328.967115:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766329.991061:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766329.991071:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766329.991079:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766329.991090:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2218 00010000:00010000:1.0:1775766329.991102:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766329.991104:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766329.991115:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766329.991159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741949824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766329.991194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766329.991198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766329.991201:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766329.991205:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2218 00000100:00080000:0.0:1775766329.991208:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766329.991211:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766329.991221:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766329.991235:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741949952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766329.991252:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766329.991253:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741949824:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766329.991265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766329.991269:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766329.991277:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2218 00000100:00080000:0.0:1775766329.991284:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766329.991290:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766329.991299:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766329.991319:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741950080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766329.991338:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766329.991352:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766329.991356:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766329.991359:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766329.991365:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2218 00000100:00080000:0.0:1775766329.991371:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766329.991375:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766329.991381:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766329.991398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766329/real 1775766329] req@ffff90c203638380 x1862023741949824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766329.991399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741950208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766329.991453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741949824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766329.991470:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766329.991474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766329.991479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741949824:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766329.991504:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766329.991530:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766329.991544:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766329/real 1775766329] req@ffff90c20363a680 x1862023741949952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766329.991586:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741949952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766329.991600:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766329.991605:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766329.991610:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741949952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766329.991622:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950080:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766329.991654:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766329.991671:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766329/real 1775766329] req@ffff90c20363bb80 x1862023741950080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766329.991684:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741950080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766329.991692:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766329.991695:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766329.991698:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766329.991704:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950208:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766329.991723:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766329.991732:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766329/real 1775766329] req@ffff90c20363b100 x1862023741950208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766329.991742:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741950208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766354 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766329.991752:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766329.991754:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766329.991757:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950208:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766331.015058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766332.039002:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766333.063123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766334.087044:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766335.111007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766335.111016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766335.111020:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766335.111029:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2223 00000100:00080000:0.0:1775766335.111041:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766335.111047:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766335.111082:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741950336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766335.111099:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766335.111109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766335.111113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766335.111114:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766335.111117:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2223 00000100:00080000:0.0:1775766335.111120:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766335.111123:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766335.111132:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766335.111141:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741950464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766335.111151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766335.111154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741950336:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766335.111155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766335.111161:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766335.111173:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2223 00000100:00080000:0.0:1775766335.111175:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766335.111177:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766335.111180:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766335.111189:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741950592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766335.111195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766335.111198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766335.111199:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766335.111213:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2223 00000100:00080000:0.0:1775766335.111215:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766335.111217:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766335.111220:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766335.111226:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741950720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766335.111262:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766335.111319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766335/real 1775766335] req@ffff90c20363aa00 x1862023741950336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766335.111335:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741950336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766335.111345:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766335.111350:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766335.111353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741950336:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766335.111377:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950464:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766335.111392:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766335.111399:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766335/real 1775766335] req@ffff90c203638000 x1862023741950464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766335.111408:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741950464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766335.111413:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766335.111416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766335.111418:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766335.111422:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950592:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766335.111431:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766335.111437:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766335/real 1775766335] req@ffff90c203639180 x1862023741950592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766335.111444:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741950592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766335.111454:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766335.111456:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766335.111458:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766335.111462:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950720:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766335.111472:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766335.111477:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766335/real 1775766335] req@ffff90c203639500 x1862023741950720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766335.111483:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741950720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766335.111490:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766335.111492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766335.111494:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950720:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766336.135066:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766337.159054:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766338.183076:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766339.207073:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766340.231047:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766340.231059:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766340.231064:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766340.231076:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2228 00010000:00010000:1.0:1775766340.231080:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766340.231092:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766340.231100:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766340.231137:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741950848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766340.231162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766340.231165:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766340.231167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766340.231169:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2228 00000100:00080000:3.0:1775766340.231172:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766340.231174:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766340.231185:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766340.231198:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741950976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766340.231210:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741950848:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766340.231212:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766340.231221:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766340.231223:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766340.231228:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2228 00000100:00080000:3.0:1775766340.231232:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766340.231236:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766340.231242:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766340.231252:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741951104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766340.231260:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766340.231262:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766340.231263:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766340.231266:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2228 00000100:00080000:3.0:1775766340.231278:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766340.231280:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766340.231284:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766340.231289:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766340.231295:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741951232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766340.231335:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766340/real 1775766340] req@ffff90c218da1f80 x1862023741950848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766340.231358:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741950848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766340.231369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766340.231373:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766340.231377:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741950848:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766340.231403:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766340.231426:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766340.231487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766340/real 1775766340] req@ffff90c218da0000 x1862023741950976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766340.231502:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741950976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766340.231511:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766340.231515:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766340.231519:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741950976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766340.231526:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766340.231548:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766340.231560:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766340/real 1775766340] req@ffff90c218da0380 x1862023741951104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766340.231572:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741951104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766340.231581:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766340.231584:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766340.231588:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766340.231594:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951232:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766340.231614:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766340.231636:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766340/real 1775766340] req@ffff90c218da3100 x1862023741951232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766340.231649:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741951232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766340.231660:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766340.231663:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766340.231667:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951232:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766341.255045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766342.279085:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766343.303140:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766344.327099:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766345.350980:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766345.350989:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766345.350994:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766345.351019:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2233 00000100:00080000:3.0:1775766345.351030:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766345.351036:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766345.351062:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766345.351085:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741951360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766345.351107:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766345.351110:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766345.351111:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766345.351114:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2233 00000100:00080000:3.0:1775766345.351116:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766345.351119:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766345.351126:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766345.351135:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741951488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766345.351142:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766345.351144:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766345.351146:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766345.351148:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2233 00000100:00080000:3.0:1775766345.351150:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766345.351154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741951360:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766345.351155:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766345.351172:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766345.351178:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741951616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766345.351185:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766345.351187:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766345.351188:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766345.351190:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2233 00000100:00080000:3.0:1775766345.351192:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766345.351195:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766345.351198:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766345.351205:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741951744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766345.351226:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766345.351258:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766345/real 1775766345] req@ffff90c218da0700 x1862023741951360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766345.351272:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741951360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766345.351280:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766345.351283:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766345.351286:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741951360:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766345.351299:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951488:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766345.351320:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766345.351331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766345/real 1775766345] req@ffff90c218da3480 x1862023741951488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766345.351344:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741951488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766345.351377:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766345.351381:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766345.351385:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766345.351410:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951616:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766345.351487:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766345.351499:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766345/real 1775766345] req@ffff90c218da0a80 x1862023741951616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766345.351518:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741951616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766345.351524:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766345.351526:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766345.351528:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766345.351532:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951744:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766345.351544:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766345.351550:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766345/real 1775766345] req@ffff90c218da0e00 x1862023741951744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766345.351557:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741951744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766345.351563:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766345.351565:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766345.351567:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741951744:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766346.375064:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766347.399029:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766348.423091:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766349.447009:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766350.470954:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766350.470962:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766350.470967:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766350.470976:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2238 00000100:00080000:3.0:1775766350.470986:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766350.470993:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766350.471027:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766350.471028:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741951872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766350.471053:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766350.471056:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766350.471058:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766350.471061:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2238 00000100:00080000:3.0:1775766350.471063:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766350.471066:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766350.471074:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766350.471083:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741951872:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766350.471085:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741952000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766350.471097:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766350.471100:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766350.471101:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766350.471103:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2238 00000100:00080000:3.0:1775766350.471105:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766350.471108:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766350.471110:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766350.471117:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741952128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766350.471124:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766350.471126:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766350.471127:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766350.471129:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2238 00000100:00080000:3.0:1775766350.471140:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766350.471143:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766350.471147:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766350.471151:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766350.471158:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741952256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766350.471189:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766350/real 1775766350] req@ffff90c218da1880 x1862023741951872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766350.471203:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741951872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766350.471221:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766350.471224:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766350.471227:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741951872:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766350.471245:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766350.471259:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766350.471266:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766350/real 1775766350] req@ffff90c218da3800 x1862023741952000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766350.471274:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741952000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766350.471279:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766350.471281:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766350.471283:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766350.471287:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766350.471296:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766350.471301:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766350/real 1775766350] req@ffff90c218da2d80 x1862023741952128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766350.471309:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741952128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766350.471313:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766350.471315:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766350.471317:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766350.471320:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952256:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766350.471329:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766350.471334:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766350/real 1775766350] req@ffff90c218da0e00 x1862023741952256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766350.471340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741952256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766350.471346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766350.471348:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766350.471350:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952256:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766351.495004:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766352.519059:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766353.543107:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766354.567066:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766355.590998:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766355.591007:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766355.591011:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766355.591019:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2244 00000100:00080000:3.0:1775766355.591037:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766355.591044:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766355.591067:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766355.591082:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741952384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766355.591104:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766355.591107:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766355.591108:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766355.591111:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2244 00000100:00080000:3.0:1775766355.591113:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766355.591116:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766355.591123:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766355.591131:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741952512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766355.591137:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766355.591140:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766355.591141:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766355.591144:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2244 00000100:00080000:3.0:1775766355.591146:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766355.591148:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766355.591151:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766355.591156:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741952640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766355.591161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766355.591163:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766355.591164:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766355.591167:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2244 00000100:00080000:3.0:1775766355.591169:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766355.591172:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766355.591175:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766355.591183:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741952768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766355.591206:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741952384:192.168.202.155@tcp:250:kworker.0 00000800:00000100:2.0:1775766355.591268:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766355.591308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766355/real 1775766355] req@ffff90c218da0a80 x1862023741952384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766355.591348:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741952384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766355.591357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766355.591360:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766355.591363:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741952384:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766355.591368:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766355.591385:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766355.591394:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766355/real 1775766355] req@ffff90c218da3480 x1862023741952512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766355.591402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741952512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766355.591408:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766355.591410:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766355.591412:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766355.591415:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766355.591425:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766355.591431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766355/real 1775766355] req@ffff90c218da0700 x1862023741952640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766355.591438:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741952640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766355.591443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766355.591444:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766355.591446:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766355.591450:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952768:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766355.591460:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766355.591465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766355/real 1775766355] req@ffff90c218da3100 x1862023741952768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766355.591472:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741952768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766355.591479:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766355.591480:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766355.591482:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741952768:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766356.615098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766357.639093:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766358.663073:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766359.687096:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766360.710975:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766360.710995:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766360.711000:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766360.711009:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2249 00000100:00080000:3.0:1775766360.711022:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766360.711030:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766360.711051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766360.711081:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741952896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766360.711109:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766360.711113:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766360.711115:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766360.711121:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2249 00000100:00080000:3.0:1775766360.711126:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766360.711131:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766360.711139:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741952896:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775766360.711141:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766360.711159:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741953024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766360.711181:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766360.711185:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766360.711187:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766360.711193:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2249 00000100:00080000:3.0:1775766360.711197:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766360.711202:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766360.711208:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766360.711222:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741953152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766360.711235:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766360.711239:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953024:192.168.202.155@tcp:38:kworker.0 00000100:00080000:3.0:1775766360.711242:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766360.711245:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766360.711260:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2249 00000100:00100000:2.0:1775766360.711262:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953152:192.168.202.155@tcp:38:kworker.0 00000100:00080000:3.0:1775766360.711280:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766360.711288:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766360.711295:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766360.711310:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741953280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766360.711343:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953280:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766361.735103:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766362.759006:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766363.783104:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775766363.783185:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775766363.783228:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775766363.783261:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766363.783301:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766363.783312:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766363.783327:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766363.783353:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766360/real 1775766363] req@ffff90c218da3100 x1862023741953280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766363.783385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741953280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766363.783402:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766363.783408:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766363.783413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953280:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766363.783429:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766360/real 1775766363] req@ffff90c218da0000 x1862023741953024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766363.783444:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741953024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766363.783452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766363.783456:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766363.783460:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766363.783467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766360/real 1775766363] req@ffff90c218da1f80 x1862023741953152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766363.783480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741953152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766363.783487:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766363.783490:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766363.783494:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766363.783501:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766360/real 1775766363] req@ffff90c218da0380 x1862023741952896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766385 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766363.783515:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741952896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766385 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766363.783524:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766363.783527:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766363.783531:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741952896:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775766364.807063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766365.831009:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766365.831020:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766365.831025:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766365.831038:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2254 00000100:00080000:3.0:1775766365.831050:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766365.831058:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766365.831114:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741953408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775766365.831129:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766365.831139:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766365.831143:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766365.831144:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766365.831148:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2254 00000100:00080000:3.0:1775766365.831152:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766365.831154:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766365.831165:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766365.831175:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741953536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766365.831181:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741953408:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766365.831197:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766365.831201:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766365.831202:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766365.831206:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2254 00000100:00080000:3.0:1775766365.831209:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766365.831211:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766365.831214:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766365.831241:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741953664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766365.831248:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766365.831252:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766365.831253:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766365.831257:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2254 00000100:00080000:3.0:1775766365.831286:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766365.831291:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766365.831313:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766365.831316:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766365.831332:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741953792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766365.831372:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766365/real 1775766365] req@ffff90c218da0700 x1862023741953408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766365.831394:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741953408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766365.831407:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766365.831412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766365.831417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741953408:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766365.831448:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953536:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766365.831473:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766365.831487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766365/real 1775766365] req@ffff90c218da3480 x1862023741953536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766365.831500:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741953536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766365.831510:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766365.831514:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766365.831518:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766365.831523:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766365.831545:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766365.831556:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766365/real 1775766365] req@ffff90c218da0a80 x1862023741953664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766365.831570:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741953664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766365.831578:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766365.831582:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766365.831585:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766365.831592:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953792:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766365.831614:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766365.831625:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766365/real 1775766365] req@ffff90c218da0e00 x1862023741953792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766365.831639:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741953792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766390 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766365.831652:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766365.831656:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766365.831659:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741953792:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766366.855001:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766367.879120:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766368.903020:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766369.927014:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766370.950957:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766370.950966:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766370.950971:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766370.950981:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2259 00000100:00080000:3.0:1775766370.951004:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766370.951012:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766370.951038:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766370.951052:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741953920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766370.951081:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766370.951085:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766370.951086:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766370.951089:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2259 00000100:00080000:3.0:1775766370.951092:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766370.951094:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766370.951103:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766370.951113:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741954048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766370.951123:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741953920:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766370.951124:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766370.951133:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766370.951135:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766370.951139:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2259 00000100:00080000:3.0:1775766370.951142:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766370.951144:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766370.951148:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766370.951157:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741954176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766370.951189:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766370.951191:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766370.951193:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766370.951197:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2259 00000100:00080000:3.0:1775766370.951212:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766370.951215:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766370.951220:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766370.951225:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766370.951235:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741954304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766370.951266:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766370/real 1775766370] req@ffff90c218da2d80 x1862023741953920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766370.951283:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741953920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766370.951298:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766370.951301:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766370.951304:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741953920:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766370.951324:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954048:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766370.951341:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766370.951350:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766370/real 1775766370] req@ffff90c218da3800 x1862023741954048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766370.951359:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741954048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766370.951376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766370.951378:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766370.951381:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766370.951385:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954176:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766370.951396:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766370.951402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766370/real 1775766370] req@ffff90c218da1880 x1862023741954176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766370.951410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741954176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766370.951416:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766370.951418:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766370.951420:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766370.951424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954304:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766370.951435:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766370.951441:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766370/real 1775766370] req@ffff90c218da0e00 x1862023741954304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766370.951450:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741954304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766395 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766370.951457:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766370.951459:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766370.951461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954304:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766371.975217:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766372.999011:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766374.023120:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766375.047152:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766376.071015:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766376.071026:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766376.071031:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766376.071043:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2264 00010000:00010000:3.0:1775766376.071045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766376.071057:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766376.071066:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766376.071109:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741954432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766376.071187:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766376.071190:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766376.071192:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766376.071196:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2264 00000100:00080000:0.0:1775766376.071200:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766376.071203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766376.071212:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766376.071227:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741954560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766376.071239:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766376.071245:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775766376.071245:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741954432:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766376.071247:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766376.071276:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2264 00000100:00080000:0.0:1775766376.071280:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766376.071284:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766376.071296:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766376.071311:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741954688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766376.071321:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766376.071324:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766376.071326:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766376.071331:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2264 00000100:00080000:0.0:1775766376.071335:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766376.071338:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766376.071343:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766376.071347:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766376.071359:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741954816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766376.071412:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766376/real 1775766376] req@ffff90c203638700 x1862023741954432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766376.071431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741954432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766376.071444:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766376.071449:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766376.071455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741954432:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766376.071501:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954560:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766376.071556:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766376.071577:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766376/real 1775766376] req@ffff90c203639500 x1862023741954560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766376.071595:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741954560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766376.071621:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766376.071623:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766376.071626:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766376.071631:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766376.071648:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766376.071655:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766376/real 1775766376] req@ffff90c203639180 x1862023741954688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766376.071665:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741954688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766376.071671:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766376.071673:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766376.071676:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766376.071680:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954816:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766376.071692:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766376.071699:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766376/real 1775766376] req@ffff90c203638000 x1862023741954816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766376.071708:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741954816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766376.071721:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766376.071724:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766376.071727:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741954816:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766377.095192:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766378.119150:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766379.143070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766380.167175:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766381.191029:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766381.191043:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766381.191050:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766381.191066:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2269 00000100:00080000:3.0:1775766381.191098:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766381.191108:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766381.191134:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766381.191162:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741954944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766381.191191:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766381.191194:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766381.191196:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766381.191201:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2269 00000100:00080000:3.0:1775766381.191205:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766381.191210:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766381.191223:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766381.191237:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741955072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766381.191238:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741954944:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766381.191265:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766381.191268:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766381.191269:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766381.191275:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2269 00000100:00080000:3.0:1775766381.191277:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766381.191281:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766381.191285:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766381.191295:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741955200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766381.191305:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766381.191307:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766381.191309:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766381.191313:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2269 00000100:00080000:3.0:1775766381.191317:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766381.191320:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766381.191324:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766381.191330:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766381.191339:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741955328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766381.191371:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766381/real 1775766381] req@ffff90c218da0a80 x1862023741954944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766381.191389:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741954944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766381.191398:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766381.191401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766381.191404:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741954944:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766381.191424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766381.191438:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766381.191446:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766381/real 1775766381] req@ffff90c218da3480 x1862023741955072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766381.191454:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741955072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766381.191460:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766381.191462:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766381.191465:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766381.191469:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766381.191479:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766381.191485:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766381/real 1775766381] req@ffff90c218da0700 x1862023741955200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766381.191492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741955200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766381.191497:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766381.191499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766381.191500:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766381.191504:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955328:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766381.191518:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766381.191526:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766381/real 1775766381] req@ffff90c218da0380 x1862023741955328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766381.191537:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741955328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766381.191547:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766381.191551:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766381.191555:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955328:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766382.215071:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766383.239244:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766384.263118:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766385.287108:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766386.311006:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766386.311042:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766386.311067:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766386.311086:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2274 00000100:00080000:3.0:1775766386.311106:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775766386.311114:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766386.311118:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766386.311182:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741955456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766386.311234:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766386.311238:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766386.311240:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766386.311244:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2274 00000100:00080000:3.0:1775766386.311248:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766386.311252:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766386.311263:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766386.311276:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741955584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766386.311288:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741955456:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766386.311291:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766386.311325:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766386.311327:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766386.311334:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2274 00000100:00080000:3.0:1775766386.311339:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766386.311347:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766386.311356:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766386.311376:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741955712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766386.311393:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766386.311394:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766386.311403:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766386.311406:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766386.311417:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2274 00000100:00080000:3.0:1775766386.311426:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766386.311434:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766386.311444:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766386/real 1775766386] req@ffff90c218da1f80 x1862023741955456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775766386.311446:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766386.311479:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741955456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766386.311486:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741955840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766386.311539:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766386.311551:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766386.311560:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741955456:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766386.311594:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955584:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766386.311622:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766386.311634:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766386/real 1775766386] req@ffff90c218da0000 x1862023741955584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766386.311646:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741955584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766386.311653:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766386.311656:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766386.311659:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766386.311664:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955712:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766386.311678:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766386.311684:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766386/real 1775766386] req@ffff90c218da3100 x1862023741955712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766386.311693:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741955712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766386.311705:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766386.311708:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766386.311711:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766386.311715:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955840:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766386.311727:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766386.311733:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766386/real 1775766386] req@ffff90c218da0380 x1862023741955840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766386.311742:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741955840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766386.311751:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766386.311753:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766386.311755:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741955840:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766387.335099:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766388.359022:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766389.383164:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766390.407133:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766391.431072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766391.431086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766391.431093:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766391.431107:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2279 00000100:00080000:1.0:1775766391.431123:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766391.431133:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766391.431166:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766391.431201:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741955968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766391.431239:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766391.431244:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766391.431246:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766391.431251:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2279 00000100:00080000:1.0:1775766391.431255:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766391.431259:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766391.431272:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766391.431288:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741956096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766391.431314:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766391.431318:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741955968:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766391.431320:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766391.431327:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766391.431340:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2279 00000100:00080000:1.0:1775766391.431346:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766391.431351:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766391.431357:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766391.431374:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741956224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766391.431388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766391.431393:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766391.431395:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766391.431399:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2279 00000100:00080000:1.0:1775766391.431407:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766391.431411:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766391.431416:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766391.431424:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766391.431443:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741956352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766391.431484:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766391/real 1775766391] req@ffff90c203496680 x1862023741955968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766391.431510:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741955968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766391.431523:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766391.431529:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766391.431534:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741955968:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766391.431564:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766391.431589:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766391.431603:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766391/real 1775766391] req@ffff90c203494a80 x1862023741956096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766391.431619:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741956096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766391.431629:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766391.431633:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766391.431637:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766391.431644:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766391.431662:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766391.431672:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766391/real 1775766391] req@ffff90c203495180 x1862023741956224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766391.431685:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741956224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766391.431694:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766391.431698:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766391.431701:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766391.431708:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956352:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766391.431725:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766391.431734:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766391/real 1775766391] req@ffff90c203495500 x1862023741956352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766391.431749:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741956352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766391.431760:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766391.431763:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766391.431767:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956352:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766392.455165:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766393.479164:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766394.503126:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766395.527222:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766396.551105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766396.551121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766396.551129:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766396.551150:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2285 00000100:00080000:1.0:1775766396.551191:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766396.551208:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766396.551212:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766396.551287:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741956480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766396.551344:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766396.551352:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766396.551357:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766396.551366:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2285 00000100:00080000:1.0:1775766396.551375:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766396.551385:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766396.551405:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766396.551411:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741956480:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766396.551448:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741956608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766396.551478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766396.551483:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766396.551486:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766396.551492:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2285 00000100:00080000:1.0:1775766396.551501:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766396.551507:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766396.551515:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775766396.551518:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766396.551545:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741956736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766396.551559:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766396.551564:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766396.551567:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766396.551575:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2285 00000100:00080000:1.0:1775766396.551580:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766396.551584:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766396/real 1775766396] req@ffff90c203494700 x1862023741956480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766396.551593:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766396.551616:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766396.551640:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741956480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766396.551658:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741956864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766396.551662:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766396.551680:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766396.551701:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741956480:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766396.551744:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766396.551774:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766396.551789:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766396/real 1775766396] req@ffff90c203495500 x1862023741956608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766396.551804:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741956608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766396.551822:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766396.551827:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766396.551831:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766396.551839:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956736:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766396.551857:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766396.551867:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766396/real 1775766396] req@ffff90c203495180 x1862023741956736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766396.551917:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741956736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766396.551953:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766396.551958:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766396.551962:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766396.551970:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956864:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766396.552000:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766396.552015:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766396/real 1775766396] req@ffff90c203494a80 x1862023741956864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766396.552040:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741956864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766396.552058:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766396.552064:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766396.552071:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741956864:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766397.575194:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766398.599053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766399.623113:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766400.647193:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766401.671123:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766401.671134:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766401.671146:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766401.671153:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766401.671169:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2290 00000100:00080000:0.0:1775766401.671185:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766401.671193:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766401.671250:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741956992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766401.671291:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766401.671312:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766401.671316:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766401.671323:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2290 00000100:00080000:0.0:1775766401.671330:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766401.671338:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766401.671354:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766401.671364:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741956992:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766401.671390:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741957120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766401.671410:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766401.671416:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766401.671419:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766401.671425:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2290 00000100:00080000:0.0:1775766401.671462:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766401.671471:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766401.671481:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766401.671488:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766401.671513:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741957248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766401.671534:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766401.671540:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775766401.671544:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766401/real 1775766401] req@ffff90c20363aa00 x1862023741956992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766401.671546:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766401.671564:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2290 00000100:00080000:0.0:1775766401.671574:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766401.671581:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741956992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766401.671585:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766401.671605:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775766401.671609:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766401.671617:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766401.671623:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741956992:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766401.671635:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741957376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766401.671655:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766401.671700:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766401.671717:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766401/real 1775766401] req@ffff90c20363b100 x1862023741957120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766401.671732:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741957120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766401.671742:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766401.671746:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766401.671750:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766401.671756:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766401.671773:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766401.671810:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766401/real 1775766401] req@ffff90c20363bb80 x1862023741957248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766401.671823:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741957248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766401.671834:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766401.671838:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766401.671844:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766401.671864:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957376:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766401.672009:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766401.672029:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766401/real 1775766401] req@ffff90c20363a680 x1862023741957376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766426 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766401.672050:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741957376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766426 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766401.672067:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766401.672073:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766401.672079:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957376:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766402.695021:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766403.718983:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766404.743061:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766405.767151:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766406.790984:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766406.790992:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766406.790998:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766406.791006:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2295 00000100:00080000:0.0:1775766406.791018:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766406.791024:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766406.791051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766406.791061:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741957504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766406.791089:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766406.791094:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766406.791096:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766406.791100:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2295 00000100:00080000:0.0:1775766406.791105:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766406.791111:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766406.791120:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766406.791122:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741957504:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766406.791140:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741957632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766406.791148:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766406.791151:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766406.791152:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766406.791155:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2295 00000100:00080000:0.0:1775766406.791158:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766406.791161:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766406.791165:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766406.791172:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741957760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766406.791179:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766406.791183:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766406.791185:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775766406.791185:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766406.791191:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2295 00000100:00080000:0.0:1775766406.791194:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766406.791197:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766406.791201:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766406.791208:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741957888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766406.791226:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766406/real 1775766406] req@ffff90c203638380 x1862023741957504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766406.791241:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741957504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766406.791249:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766406.791252:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766406.791255:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741957504:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766406.791272:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766406.791286:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766406.791293:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766406/real 1775766406] req@ffff90c203639f80 x1862023741957632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766406.791301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741957632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766406.791306:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766406.791309:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766406.791311:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766406.791314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766406.791323:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766406.791328:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766406/real 1775766406] req@ffff90c203638e00 x1862023741957760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766406.791335:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741957760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766406.791340:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766406.791342:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766406.791343:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766406.791347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957888:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766406.791357:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766406.791362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766406/real 1775766406] req@ffff90c20363b480 x1862023741957888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766431 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766406.791369:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741957888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766431 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766406.791374:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766406.791376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766406.791378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741957888:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766407.814980:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766408.839069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766409.863035:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766410.887047:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766411.910980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766411.910989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766411.910994:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766411.911003:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2300 00000100:00080000:1.0:1775766411.911013:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766411.911020:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766411.911025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766411.911058:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741958016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766411.911084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766411.911087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766411.911089:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766411.911092:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2300 00000100:00080000:1.0:1775766411.911094:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766411.911096:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766411.911105:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766411.911114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741958144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766411.911123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766411.911140:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741958016:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766411.911146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766411.911152:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766411.911158:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2300 00000100:00080000:1.0:1775766411.911162:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766411.911166:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766411.911171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766411.911183:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741958272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766411.911193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766411.911196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766411.911198:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766411.911202:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2300 00000100:00080000:1.0:1775766411.911206:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766411.911209:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766411.911214:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766411.911225:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741958400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766411.911249:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766411.911289:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766411/real 1775766411] req@ffff90c203496680 x1862023741958016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766411.911304:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741958016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766411.911312:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766411.911316:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766411.911319:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741958016:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766411.911339:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766411.911354:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766411.911362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766411/real 1775766411] req@ffff90c203496a00 x1862023741958144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766411.911371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741958144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766411.911377:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766411.911379:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766411.911381:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766411.911385:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958272:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766411.911395:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766411.911411:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766411/real 1775766411] req@ffff90c203496300 x1862023741958272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766411.911419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741958272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766411.911424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766411.911426:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766411.911428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766411.911431:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958400:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766411.911462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766411.911471:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766411/real 1775766411] req@ffff90c203497b80 x1862023741958400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766411.911481:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741958400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766436 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766411.911497:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766411.911500:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766411.911503:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958400:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766412.935019:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766413.959047:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766414.983086:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766416.007086:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766417.030962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766417.030971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766417.030975:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766417.030984:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2305 00000100:00080000:1.0:1775766417.030995:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766417.031001:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766417.031029:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766417.031038:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741958528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766417.031060:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766417.031063:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766417.031065:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766417.031067:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2305 00000100:00080000:1.0:1775766417.031070:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766417.031072:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766417.031081:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766417.031093:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741958656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766417.031094:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741958528:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766417.031107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766417.031110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766417.031111:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766417.031113:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2305 00000100:00080000:1.0:1775766417.031115:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766417.031117:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766417.031120:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766417.031127:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741958784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766417.031132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766417.031134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766417.031135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766417.031138:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2305 00000100:00080000:1.0:1775766417.031141:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766417.031143:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766417.031146:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766417.031153:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741958912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766417.031165:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766417.031207:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766417/real 1775766417] req@ffff90c203496d80 x1862023741958528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766417.031222:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741958528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766417.031230:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766417.031233:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766417.031236:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741958528:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766417.031254:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766417.031270:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766417.031278:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766417/real 1775766417] req@ffff90c203495880 x1862023741958656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766417.031286:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741958656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766417.031291:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766417.031294:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766417.031296:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766417.031300:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766417.031310:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766417.031315:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766417/real 1775766417] req@ffff90c203497100 x1862023741958784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766417.031323:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741958784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766417.031331:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766417.031334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766417.031338:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766417.031343:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958912:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766417.031363:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766417.031373:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766417/real 1775766417] req@ffff90c203495c00 x1862023741958912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766417.031385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741958912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766417.031395:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766417.031397:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766417.031400:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741958912:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766418.055145:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766419.079049:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766420.103076:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766421.127005:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766422.150984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766422.150993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766422.150998:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766422.151018:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2310 00000100:00080000:1.0:1775766422.151028:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766422.151062:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766422.151063:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766422.151101:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741959040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766422.151122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766422.151125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766422.151126:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766422.151129:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2310 00000100:00080000:1.0:1775766422.151131:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766422.151134:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766422.151141:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766422.151149:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741959168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766422.151157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766422.151161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766422.151163:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766422.151165:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741959040:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766422.151167:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2310 00000100:00080000:1.0:1775766422.151172:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766422.151175:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766422.151179:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766422.151185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741959296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766422.151193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766422.151195:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766422.151196:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766422.151199:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2310 00000100:00080000:1.0:1775766422.151201:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766422.151203:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766422.151206:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766422.151212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741959424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766422.151249:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766422.151287:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766422/real 1775766422] req@ffff90c203494380 x1862023741959040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766422.151306:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741959040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766422.151316:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766422.151320:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766422.151325:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741959040:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766422.151357:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766422.151381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766422.151393:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766422/real 1775766422] req@ffff90c203495f80 x1862023741959168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766422.151406:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741959168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766422.151414:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766422.151418:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766422.151421:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766422.151428:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766422.151446:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766422.151457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766422/real 1775766422] req@ffff90c203494000 x1862023741959296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766422.151469:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741959296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766422.151483:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766422.151487:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766422.151490:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766422.151496:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959424:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766422.151516:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766422.151527:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766422/real 1775766422] req@ffff90c203497480 x1862023741959424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766422.151538:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741959424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766422.151548:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766422.151550:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766422.151554:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959424:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766423.176304:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766424.199050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766425.223025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766426.247038:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766427.270970:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766427.270978:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766427.270982:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766427.270991:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2315 00000100:00080000:0.0:1775766427.271002:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766427.271008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766427.271009:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766427.271046:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741959552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766427.271071:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766427.271085:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766427.271087:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766427.271092:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2315 00000100:00080000:0.0:1775766427.271096:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766427.271101:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766427.271111:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766427.271120:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741959552:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766427.271129:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741959680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766427.271147:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766427.271150:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766427.271152:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766427.271154:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2315 00000100:00080000:0.0:1775766427.271159:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766427.271162:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766427.271166:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766427.271175:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741959808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766427.271182:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766427.271185:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766427.271186:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766427.271189:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2315 00000100:00080000:0.0:1775766427.271191:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766427.271194:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766427.271197:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766427.271204:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741959936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766427.271240:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766427.271266:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766427.271281:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959936:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766428.295053:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766429.318994:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766430.343027:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766430.343113:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775766430.343146:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766430.343161:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766430.343189:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766430.343198:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766430.343207:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766430.343220:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766427/real 1775766430] req@ffff90c20363b480 x1862023741959680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766430.343241:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741959680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766430.343250:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766430.343254:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766430.343257:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766430.343262:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766427/real 1775766430] req@ffff90c203638e00 x1862023741959808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766430.343269:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741959808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766430.343274:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766430.343276:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766430.343278:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766430.343281:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766427/real 1775766430] req@ffff90c203639f80 x1862023741959936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766430.343288:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741959936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766430.343294:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766430.343297:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766430.343299:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741959936:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766430.343302:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766427/real 1775766430] req@ffff90c20363b800 x1862023741959552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766430.343309:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741959552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766430.343314:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766430.343316:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766430.343318:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741959552:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766431.367012:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766432.391015:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766432.391029:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766432.391036:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766432.391048:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2320 00000100:00080000:0.0:1775766432.391064:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766432.391074:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766432.391099:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766432.391125:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741960064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766432.391158:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766432.391165:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766432.391168:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766432.391174:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2320 00000100:00080000:0.0:1775766432.391181:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766432.391187:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766432.391201:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766432.391214:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741960064:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766432.391221:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741960192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766432.391236:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766432.391241:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766432.391243:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766432.391249:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2320 00000100:00080000:0.0:1775766432.391254:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766432.391258:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766432.391265:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766432.391286:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741960320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766432.391299:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775766432.391304:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766432.391305:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766432.391308:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766432.391313:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2320 00000100:00080000:0.0:1775766432.391318:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766432.391323:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766432.391332:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766432.391345:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741960448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766432.391354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766432/real 1775766432] req@ffff90c203638380 x1862023741960064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766432.391383:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741960064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766432.391399:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766432.391405:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766432.391412:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741960064:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766432.391444:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766432.391482:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766432.391503:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766432/real 1775766432] req@ffff90c20363a680 x1862023741960192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766432.391521:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741960192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766432.391534:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766432.391540:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766432.391544:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766432.391599:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766432.391635:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766432.391654:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766432/real 1775766432] req@ffff90c20363bb80 x1862023741960320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766432.391672:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741960320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766432.391683:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766432.391687:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766432.391692:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766432.391701:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960448:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766432.391729:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766432.391746:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766432/real 1775766432] req@ffff90c20363b100 x1862023741960448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766432.391763:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741960448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766432.391777:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766432.391782:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766432.391786:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960448:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766433.415078:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766434.438993:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766435.463023:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766436.487045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766437.510955:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766437.510967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766437.510972:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766437.510980:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2325 00000100:00080000:0.0:1775766437.510989:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766437.510995:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766437.511031:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741960576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766437.511033:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766437.511054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766437.511057:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766437.511058:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766437.511061:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2325 00000100:00080000:0.0:1775766437.511063:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766437.511065:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766437.511073:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766437.511081:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741960704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766437.511089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766437.511091:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766437.511093:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766437.511097:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2325 00000100:00100000:2.0:1775766437.511098:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741960576:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766437.511100:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766437.511106:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766437.511113:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766437.511120:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741960832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766437.511127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766437.511129:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766437.511130:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766437.511133:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2325 00000100:00080000:0.0:1775766437.511135:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766437.511137:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766437.511140:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766437.511147:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741960960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766437.511174:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766437.511209:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766437/real 1775766437] req@ffff90c20363aa00 x1862023741960576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766437.511225:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741960576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766437.511232:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766437.511235:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766437.511238:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741960576:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766437.511257:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766437.511283:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766437.511292:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766437/real 1775766437] req@ffff90c203638000 x1862023741960704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766437.511300:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741960704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766437.511306:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766437.511308:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766437.511310:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766437.511314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960832:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766437.511325:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766437.511330:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766437/real 1775766437] req@ffff90c203639180 x1862023741960832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766437.511338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741960832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766437.511347:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766437.511349:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766437.511352:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766437.511355:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960960:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766437.511364:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766437.511369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766437/real 1775766437] req@ffff90c203639500 x1862023741960960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766437.511376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741960960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766437.511388:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766437.511391:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766437.511393:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741960960:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766438.535080:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766439.559008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766440.583044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766441.607014:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766442.631000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766442.631008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766442.631012:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766442.631022:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2331 00000100:00080000:1.0:1775766442.631032:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766442.631040:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766442.631054:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766442.631078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741961088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766442.631104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766442.631108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766442.631109:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766442.631112:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2331 00000100:00080000:1.0:1775766442.631114:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766442.631116:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766442.631126:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766442.631142:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741961216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766442.631172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766442.631181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766442.631185:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766442.631185:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741961088:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766442.631194:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2331 00000100:00080000:1.0:1775766442.631202:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766442.631208:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766442.631216:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766442.631233:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741961344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766442.631243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766442.631264:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766442.631266:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766442.631271:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2331 00000100:00080000:1.0:1775766442.631275:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766442.631279:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766442.631285:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766442.631294:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741961472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766442.631326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766442.631392:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766442/real 1775766442] req@ffff90c203494e00 x1862023741961088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766442.631416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741961088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766442.631466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766442.631472:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766442.631476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741961088:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766442.631506:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961216:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766442.631530:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766442.631543:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766442/real 1775766442] req@ffff90c203497480 x1862023741961216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766442.631556:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741961216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766442.631564:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766442.631567:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766442.631571:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766442.631577:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766442.631594:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766442.631603:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766442/real 1775766442] req@ffff90c203494000 x1862023741961344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766442.631614:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741961344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766442.631622:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766442.631625:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766442.631629:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766442.631635:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961472:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766442.631652:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766442.631661:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766442/real 1775766442] req@ffff90c203495f80 x1862023741961472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766467 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766442.631673:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741961472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766467 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766442.631684:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766442.631687:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766442.631690:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961472:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766443.655105:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766444.679063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766445.703105:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766446.726999:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766447.751017:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766447.751025:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766447.751030:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766447.751039:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2336 00000100:00080000:3.0:1775766447.751051:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766447.751064:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766447.751087:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766447.751101:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741961600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766447.751125:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766447.751128:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766447.751129:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766447.751132:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2336 00000100:00080000:3.0:1775766447.751134:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766447.751137:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766447.751146:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766447.751155:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741961728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766447.751164:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766447.751168:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766447.751171:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766447.751175:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2336 00000100:00080000:3.0:1775766447.751177:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766447.751182:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766447.751184:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741961600:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775766447.751186:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766447.751198:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741961856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766447.751204:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766447.751206:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766447.751207:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766447.751211:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2336 00000100:00080000:3.0:1775766447.751213:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766447.751216:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766447.751218:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766447.751228:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741961984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766447.751281:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766447.751325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766447/real 1775766447] req@ffff90c218da0700 x1862023741961600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766447.751341:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741961600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766447.751351:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766447.751354:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766447.751358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741961600:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766447.751381:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766447.751434:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766447.751444:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766447/real 1775766447] req@ffff90c218da3480 x1862023741961728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766447.751453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741961728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766447.751464:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766447.751467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766447.751469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766447.751473:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766447.751486:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766447.751492:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766447/real 1775766447] req@ffff90c218da0a80 x1862023741961856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766447.751499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741961856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766447.751504:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766447.751506:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766447.751508:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766447.751512:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961984:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766447.751523:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766447.751528:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766447/real 1775766447] req@ffff90c218da0e00 x1862023741961984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766472 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766447.751536:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741961984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766472 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766447.751544:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766447.751547:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766447.751551:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741961984:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766448.775092:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766449.799091:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766450.823192:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766451.847188:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766452.871017:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766452.871028:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766452.871033:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766452.871044:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2341 00000100:00080000:1.0:1775766452.871059:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766452.871064:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766452.871068:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766452.871108:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741962112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766452.871140:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766452.871143:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766452.871145:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766452.871147:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2341 00000100:00080000:1.0:1775766452.871151:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766452.871155:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766452.871163:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766452.871173:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741962240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766452.871182:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766452.871188:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766452.871190:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766452.871190:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741962112:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766452.871208:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2341 00000100:00080000:1.0:1775766452.871213:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766452.871218:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766452.871225:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766452.871242:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741962368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766452.871254:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766452.871258:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766452.871260:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766452.871265:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2341 00000100:00080000:1.0:1775766452.871270:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766452.871274:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766452.871281:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766452.871282:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766452.871305:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741962496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766452.871344:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766452/real 1775766452] req@ffff90c203494380 x1862023741962112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766452.871371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741962112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766452.871385:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766452.871390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766452.871395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741962112:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766452.871424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962240:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766452.871443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766452.871452:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766452/real 1775766452] req@ffff90c203495c00 x1862023741962240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766452.871461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741962240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766452.871467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766452.871469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766452.871471:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766452.871475:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962368:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766452.871486:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766452.871492:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766452/real 1775766452] req@ffff90c203497100 x1862023741962368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766452.871500:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741962368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766452.871505:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766452.871506:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766452.871508:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766452.871512:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962496:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766452.871522:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766452.871528:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766452/real 1775766452] req@ffff90c203495880 x1862023741962496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766452.871534:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741962496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766477 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766452.871541:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766452.871543:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766452.871545:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962496:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766453.895069:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766454.919118:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766455.943021:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766456.967139:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766457.991013:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766457.991023:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766457.991027:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766457.991037:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2346 00000100:00080000:1.0:1775766457.991048:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766457.991048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766457.991068:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766457.991103:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741962624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766457.991138:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766457.991141:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766457.991142:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766457.991145:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2346 00000100:00080000:1.0:1775766457.991147:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766457.991149:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766457.991157:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766457.991168:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741962752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766457.991175:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766457.991178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766457.991180:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766457.991183:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2346 00000100:00080000:1.0:1775766457.991186:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766457.991188:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766457.991192:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766457.991201:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741962624:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766457.991205:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741962880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766457.991224:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766457.991226:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766457.991227:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766457.991230:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2346 00000100:00080000:1.0:1775766457.991233:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766457.991236:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766457.991241:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766457.991252:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741963008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766457.991326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766457.991385:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766457/real 1775766457] req@ffff90c203496d80 x1862023741962624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766457.991406:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741962624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766457.991418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766457.991422:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766457.991427:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741962624:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766457.991436:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766457.991467:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766457.991485:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766457/real 1775766457] req@ffff90c203497b80 x1862023741962752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766457.991503:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741962752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766457.991515:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766457.991520:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766457.991524:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766457.991556:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766457.991578:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766457.991588:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766457/real 1775766457] req@ffff90c203496300 x1862023741962880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766457.991615:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741962880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766457.991624:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766457.991628:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766457.991631:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741962880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766457.991650:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963008:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766457.991669:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766457.991678:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766457/real 1775766457] req@ffff90c203496a00 x1862023741963008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766457.991689:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741963008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766482 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766457.991700:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766457.991703:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766457.991706:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963008:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766459.015095:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766460.039076:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766461.063177:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766462.087080:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766463.111069:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766463.111083:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766463.111089:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766463.111105:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2351 00010000:00010000:1.0:1775766463.111115:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766463.111122:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766463.111134:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766463.111195:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741963136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766463.111250:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766463.111284:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766463.111287:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766463.111293:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2351 00000100:00080000:0.0:1775766463.111298:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766463.111303:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766463.111316:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766463.111326:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741963136:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766463.111337:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741963264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766463.111350:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766463.111354:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766463.111356:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766463.111360:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2351 00000100:00080000:0.0:1775766463.111364:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766463.111368:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766463.111373:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766463.111383:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741963392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766463.111393:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766463.111397:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766463.111399:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766463.111403:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2351 00000100:00080000:0.0:1775766463.111407:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766463.111413:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766463.111418:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766463.111442:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766463.111458:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741963520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766463.111484:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766463/real 1775766463] req@ffff90c203638700 x1862023741963136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766463.111503:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741963136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766463.111513:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766463.111517:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766463.111521:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741963136:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766463.111545:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766463.111581:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766463.111595:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766463/real 1775766463] req@ffff90c203639500 x1862023741963264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766463.111614:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741963264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766463.111637:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766463.111641:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766463.111645:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766463.111650:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963392:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766463.111680:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766463.111689:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766463/real 1775766463] req@ffff90c203639180 x1862023741963392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766463.111699:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741963392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766463.111710:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766463.111713:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766463.111716:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766463.111721:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963520:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766463.111735:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766463.111741:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766463/real 1775766463] req@ffff90c203638000 x1862023741963520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766463.111750:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741963520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766463.111764:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766463.111767:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766463.111770:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963520:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766464.135087:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766465.159169:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766466.183105:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766467.207161:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766468.230967:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766468.230976:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766468.230980:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766468.230988:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2356 00000100:00080000:1.0:1775766468.231000:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766468.231006:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766468.231042:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741963648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766468.231063:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766468.231067:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766468.231070:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766468.231075:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2356 00000100:00080000:1.0:1775766468.231078:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766468.231083:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766468.231084:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766468.231093:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766468.231106:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741963776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766468.231114:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766468.231118:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766468.231119:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766468.231130:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2356 00000100:00100000:2.0:1775766468.231132:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741963648:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766468.231134:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766468.231144:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766468.231147:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766468.231159:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741963904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766468.231168:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766468.231191:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766468.231193:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766468.231197:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2356 00000100:00080000:1.0:1775766468.231201:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766468.231204:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766468.231210:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766468.231218:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741964032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766468.231238:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766468.231282:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766468/real 1775766468] req@ffff90c203496680 x1862023741963648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766468.231298:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741963648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766468.231307:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766468.231311:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766468.231314:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741963648:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766468.231320:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963776:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766468.231335:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766468.231342:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766468/real 1775766468] req@ffff90c203494a80 x1862023741963776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766468.231350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741963776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766468.231355:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766468.231357:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766468.231359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766468.231383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963904:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766468.231394:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766468.231400:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766468/real 1775766468] req@ffff90c203495180 x1862023741963904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766468.231408:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741963904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766468.231412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766468.231414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766468.231416:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741963904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766468.231420:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964032:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766468.231430:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766468.231436:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766468/real 1775766468] req@ffff90c203495500 x1862023741964032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766468.231443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741964032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766468.231450:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766468.231452:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766468.231454:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964032:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766469.255091:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766470.279082:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766471.303158:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766472.327113:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766473.350966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766473.350977:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766473.350981:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766473.350994:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2361 00000100:00080000:2.0:1775766473.351016:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766473.351024:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766473.351068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bfb80 x1862023741964160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766473.351097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766473.351100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766473.351101:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766473.351105:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2361 00000100:00080000:2.0:1775766473.351108:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766473.351113:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766473.351148:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766473.351152:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741964160:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766473.351168:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd500 x1862023741964288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766473.351181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766473.351184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766473.351186:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766473.351189:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2361 00000100:00080000:2.0:1775766473.351193:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766473.351196:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766473.351201:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766473.351212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023741964416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766473.351225:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775766473.351229:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775766473.351230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766473.351233:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766473.351237:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2361 00000100:00080000:2.0:1775766473.351240:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766473.351243:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766473.351247:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766473.351255:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023741964544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766473.351278:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766473/real 1775766473] req@ffff90c2034bfb80 x1862023741964160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766473.351295:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bfb80 x1862023741964160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766473.351304:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766473.351308:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766473.351312:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741964160:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766473.351339:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964288:192.168.202.155@tcp:38:kworker.0 00010000:00010000:3.0:1775766473.351354:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1775766473.351359:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766473.351374:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766473/real 1775766473] req@ffff90c2034bd500 x1862023741964288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766473.351384:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd500 x1862023741964288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766473.351389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766473.351392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766473.351394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766473.351398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766473.351409:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766473.351415:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766473/real 1775766473] req@ffff90c2034be300 x1862023741964416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766473.351423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023741964416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766473.351427:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766473.351429:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766473.351431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964416:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766473.351434:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964544:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766473.351444:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766473.351448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766473/real 1775766473] req@ffff90c2034bf480 x1862023741964544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766473.351455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023741964544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766473.351462:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766473.351464:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766473.351466:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964544:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766474.375082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766475.399296:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766476.423195:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766477.447160:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766478.471106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766478.471119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766478.471126:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766478.471140:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2366 00000100:00080000:2.0:1775766478.471156:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766478.471164:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766478.471165:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766478.471228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023741964672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766478.471259:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766478.471263:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766478.471265:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766478.471269:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2366 00000100:00080000:2.0:1775766478.471272:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766478.471275:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766478.471287:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766478.471302:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023741964800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766478.471313:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766478.471318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766478.471320:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766478.471323:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2366 00000100:00080000:2.0:1775766478.471326:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766478.471329:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766478.471333:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766478.471342:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741964672:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766478.471344:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023741964928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766478.471377:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766478.471380:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766478.471382:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766478.471386:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2366 00000100:00080000:2.0:1775766478.471389:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766478.471392:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766478.471397:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766478.471407:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023741965056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766478.471465:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766478.471518:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766478/real 1775766478] req@ffff90c2034bea00 x1862023741964672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766478.471558:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023741964672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766478.471573:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766478.471579:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766478.471585:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741964672:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766478.471597:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766478.471630:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766478.471647:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766478/real 1775766478] req@ffff90c2034bca80 x1862023741964800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766478.471665:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023741964800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766478.471677:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766478.471681:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766478.471686:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766478.471719:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964928:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766478.471747:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766478.471763:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766478/real 1775766478] req@ffff90c2034bce00 x1862023741964928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766478.471778:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023741964928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766478.471797:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766478.471802:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766478.471807:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741964928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766478.471816:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965056:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766478.471844:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766478.471860:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766478/real 1775766478] req@ffff90c2034bc380 x1862023741965056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766478.471921:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023741965056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766478.471938:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766478.471944:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766478.471950:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965056:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766479.495098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766480.519053:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766481.543091:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766482.567229:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766483.591028:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766483.591043:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766483.591049:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766483.591063:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2372 00000100:00080000:1.0:1775766483.591090:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766483.591099:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766483.591134:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766483.591142:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741965184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766483.591174:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766483.591178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766483.591180:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766483.591184:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2372 00000100:00080000:1.0:1775766483.591187:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766483.591190:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766483.591200:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766483.591214:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741965312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766483.591226:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775766483.591228:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741965184:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766483.591235:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766483.591240:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766483.591245:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2372 00000100:00080000:1.0:1775766483.591248:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766483.591251:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766483.591256:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766483.591266:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741965440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766483.591275:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766483.591278:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766483.591280:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766483.591284:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2372 00000100:00080000:1.0:1775766483.591290:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766483.591294:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766483.591298:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775766483.591307:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766483.591312:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741965568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766483.591358:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766483/real 1775766483] req@ffff90c203494700 x1862023741965184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766483.591376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741965184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766483.591386:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766483.591390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766483.591394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741965184:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766483.591415:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766483.591435:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766483.591445:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766483/real 1775766483] req@ffff90c203495500 x1862023741965312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766483.591460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741965312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766483.591472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766483.591476:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766483.591482:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766483.591490:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766483.591518:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766483.591533:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766483/real 1775766483] req@ffff90c203495180 x1862023741965440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766483.591550:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741965440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766483.591557:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766483.591560:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766483.591563:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766483.591569:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965568:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766483.591589:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766483.591597:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766483/real 1775766483] req@ffff90c203494a80 x1862023741965568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766483.591608:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741965568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766483.591617:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766483.591619:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766483.591622:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965568:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766484.615176:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766485.639111:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766486.663157:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766487.687131:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766488.711011:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766488.711019:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766488.711023:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766488.711032:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2377 00000100:00080000:0.0:1775766488.711045:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766488.711051:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766488.711088:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741965696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766488.711110:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766488.711115:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766488.711119:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766488.711121:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766488.711124:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2377 00000100:00080000:0.0:1775766488.711126:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766488.711129:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766488.711137:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766488.711148:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741965824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766488.711168:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766488.711171:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766488.711173:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766488.711180:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2377 00000100:00080000:0.0:1775766488.711184:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766488.711187:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766488.711189:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741965696:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775766488.711192:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766488.711201:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741965952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766488.711222:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766488.711225:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766488.711226:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766488.711228:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2377 00000100:00080000:0.0:1775766488.711247:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766488.711251:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766488.711257:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766488.711269:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741966080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766488.711329:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766488.711388:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766488/real 1775766488] req@ffff90c20363aa00 x1862023741965696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766488.711441:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741965696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766488.711465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766488.711474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766488.711483:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741965696:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766488.711529:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766488.711572:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766488.711595:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766488/real 1775766488] req@ffff90c20363b100 x1862023741965824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766488.711642:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741965824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766488.711660:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766488.711668:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766488.711674:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766488.711688:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766488.711745:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766488.711759:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766488/real 1775766488] req@ffff90c20363bb80 x1862023741965952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766488.711786:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741965952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766488.711797:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766488.711802:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766488.711807:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741965952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766488.711816:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966080:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766488.711842:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766488.711856:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766488/real 1775766488] req@ffff90c20363a680 x1862023741966080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766513 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766488.711872:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741966080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766513 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766488.711887:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766488.711926:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766488.711931:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966080:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766489.735204:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766490.759069:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766491.783301:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766492.807139:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766493.831041:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766493.831056:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766493.831062:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766493.831077:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2382 00000100:00080000:0.0:1775766493.831092:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766493.831099:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766493.831118:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766493.831147:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741966208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766493.831172:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766493.831175:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766493.831177:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766493.831180:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2382 00000100:00080000:0.0:1775766493.831183:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766493.831186:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766493.831196:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766493.831208:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741966208:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766493.831209:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741966336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766493.831222:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766493.831224:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766493.831226:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766493.831229:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2382 00000100:00080000:0.0:1775766493.831233:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766493.831237:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766493.831243:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766493.831257:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741966464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766493.831269:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766493.831273:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766493.831275:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766493.831280:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2382 00000100:00080000:0.0:1775766493.831285:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766493.831289:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766493.831294:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766493.831307:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741966592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766493.831322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766493.831348:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766493.831363:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966592:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766494.855082:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766495.879057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766496.903186:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1775766496.903205:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:2.0:1775766496.903269:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1775766496.903286:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:2.0:1775766496.903343:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:2.0:1775766496.903365:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:2.0:1775766496.903374:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775766496.903421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766493/real 1775766496] req@ffff90c20363b800 x1862023741966336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766496.903455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741966336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766496.903468:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766496.903474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766496.903479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766496.903487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766493/real 1775766496] req@ffff90c203639f80 x1862023741966464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766496.903501:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741966464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766496.903509:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766496.903513:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766496.903516:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766496.903526:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766493/real 1775766496] req@ffff90c203638e00 x1862023741966592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766496.903540:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741966592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766496.903550:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766496.903554:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766496.903558:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966592:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775766496.903566:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766493/real 1775766496] req@ffff90c203638380 x1862023741966208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766496.903579:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741966208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766518 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766496.903587:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766496.903591:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766496.903594:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741966208:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766497.927011:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766498.951070:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766498.951086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766498.951093:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775766498.951109:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766498.951110:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2387 00000100:00080000:0.0:1775766498.951130:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766498.951143:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766498.951209:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741966720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766498.951271:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766498.951278:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766498.951282:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766498.951290:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2387 00000100:00080000:0.0:1775766498.951299:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766498.951307:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766498.951323:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766498.951334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741966720:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766498.951355:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741966848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766498.951377:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766498.951383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766498.951387:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766498.951394:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2387 00000100:00080000:0.0:1775766498.951402:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766498.951408:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766498.951417:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766498.951438:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741966976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766498.951445:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766498.951465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766498.951471:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766498.951475:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766498.951484:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2387 00000100:00080000:0.0:1775766498.951492:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766498.951499:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766498.951520:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766498.951527:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766498/real 1775766498] req@ffff90c203638380 x1862023741966720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766498.951546:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741967104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766498.951577:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741966720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766498.951598:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766498.951606:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766498.951615:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741966720:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766498.951651:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766498.951692:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766498.951715:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766498/real 1775766498] req@ffff90c203638e00 x1862023741966848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766498.951738:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741966848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766498.951753:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766498.951759:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766498.951764:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766498.951775:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766498.951808:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766498.951827:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766498/real 1775766498] req@ffff90c203639f80 x1862023741966976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766498.951845:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741966976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766498.951920:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766498.951927:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766498.951933:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741966976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766498.951945:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967104:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766498.951984:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766498.952005:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766498/real 1775766498] req@ffff90c20363b800 x1862023741967104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766498.952028:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741967104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766523 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766498.952046:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766498.952052:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766498.952058:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967104:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766499.975100:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766500.999015:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766502.023020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766503.047006:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766504.070997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766504.071011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766504.071018:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766504.071033:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2392 00010000:00010000:3.0:1775766504.071037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766504.071049:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766504.071059:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766504.071110:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741967232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766504.071135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766504.071139:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766504.071140:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766504.071143:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2392 00000100:00080000:0.0:1775766504.071148:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766504.071152:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766504.071162:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766504.071179:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741967360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766504.071179:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741967232:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766504.071202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766504.071206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766504.071207:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766504.071212:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2392 00000100:00080000:0.0:1775766504.071214:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766504.071218:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766504.071222:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766504.071238:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741967488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766504.071257:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766504.071258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766504.071263:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766504.071266:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766504.071273:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2392 00000100:00080000:0.0:1775766504.071279:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766504.071284:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766504.071291:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766504.071295:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766504/real 1775766504] req@ffff90c20363b480 x1862023741967232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766504.071309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741967616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766504.071314:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741967232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766504.071371:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766504.071375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766504.071379:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741967232:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766504.071404:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766504.071419:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766504.071427:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766504/real 1775766504] req@ffff90c20363b800 x1862023741967360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766504.071435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741967360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766504.071441:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766504.071445:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766504.071448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766504.071455:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967488:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766504.071473:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766504.071484:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766504/real 1775766504] req@ffff90c203639f80 x1862023741967488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766504.071497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741967488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766504.071520:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766504.071523:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766504.071525:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766504.071530:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967616:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766504.071546:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766504.071553:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766504/real 1775766504] req@ffff90c203638e00 x1862023741967616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766504.071560:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741967616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766504.071567:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766504.071569:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766504.071571:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967616:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766505.094980:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766506.119011:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766507.143041:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766508.167007:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766509.191012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766509.191020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766509.191025:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766509.191033:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2397 00010000:00010000:3.0:1775766509.191035:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766509.191044:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766509.191050:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766509.191097:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741967744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766509.191127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766509.191130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766509.191132:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766509.191135:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2397 00000100:00080000:0.0:1775766509.191139:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766509.191142:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766509.191150:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766509.191159:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741967744:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766509.191164:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741967872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766509.191173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766509.191176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766509.191177:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766509.191180:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2397 00000100:00080000:0.0:1775766509.191182:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766509.191185:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766509.191194:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766509.191201:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741968000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766509.191207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766509.191210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766509.191211:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766509.191214:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2397 00000100:00080000:0.0:1775766509.191217:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766509.191222:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766509.191227:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:0.0:1775766509.191228:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766509.191237:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741968128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766509.191272:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766509/real 1775766509] req@ffff90c203638380 x1862023741967744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766509.191289:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741967744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766509.191296:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766509.191300:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766509.191302:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741967744:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766509.191321:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967872:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766509.191338:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766509.191347:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766509/real 1775766509] req@ffff90c20363a680 x1862023741967872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766509.191355:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741967872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766509.191360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766509.191362:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766509.191364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741967872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766509.191368:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766509.191377:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766509.191383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766509/real 1775766509] req@ffff90c20363bb80 x1862023741968000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766509.191390:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741968000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766509.191396:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766509.191399:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766509.191402:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766509.191408:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968128:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766509.191425:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766509.191435:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766509/real 1775766509] req@ffff90c20363b100 x1862023741968128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766509.191448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741968128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766509.191459:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766509.191462:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766509.191465:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968128:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766510.215066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766511.239023:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766512.263033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766513.287077:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766514.310981:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766514.310991:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766514.310996:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766514.311007:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2402 00000100:00080000:1.0:1775766514.311019:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766514.311027:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766514.311032:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766514.311069:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741968256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766514.311098:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766514.311102:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766514.311104:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766514.311109:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2402 00000100:00080000:1.0:1775766514.311113:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766514.311117:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766514.311128:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766514.311129:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741968256:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766514.311146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741968384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766514.311158:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766514.311162:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766514.311164:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766514.311168:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2402 00000100:00080000:1.0:1775766514.311172:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766514.311177:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766514.311181:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766514.311196:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741968512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766514.311196:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766514.311209:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766514.311213:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766514.311215:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766514.311220:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2402 00000100:00080000:1.0:1775766514.311226:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766514.311229:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766514/real 1775766514] req@ffff90c203496680 x1862023741968256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766514.311234:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766514.311245:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766514.311250:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741968256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766514.311259:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775766514.311260:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741968640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766514.311264:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766514.311272:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741968256:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766514.311296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766514.311311:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766514.311319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766514/real 1775766514] req@ffff90c203496a00 x1862023741968384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766514.311327:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741968384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766514.311337:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766514.311339:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766514.311342:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766514.311345:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766514.311355:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766514.311368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766514/real 1775766514] req@ffff90c203496300 x1862023741968512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766514.311376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741968512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766514.311381:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766514.311383:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766514.311385:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766514.311388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968640:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766514.311399:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766514.311405:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766514/real 1775766514] req@ffff90c203497b80 x1862023741968640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766514.311412:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741968640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766514.311418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766514.311420:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766514.311422:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968640:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766515.335056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766516.359065:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766517.383049:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766518.407026:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766519.430929:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766519.430937:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766519.430941:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766519.430950:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2407 00000100:00080000:1.0:1775766519.430960:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766519.430966:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766519.431003:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741968768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766519.431019:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766519.431034:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766519.431037:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766519.431038:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766519.431041:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2407 00000100:00080000:1.0:1775766519.431044:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766519.431046:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766519.431053:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766519.431064:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741968896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766519.431074:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766519.431074:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741968768:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766519.431083:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766519.431085:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766519.431088:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2407 00000100:00080000:1.0:1775766519.431091:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766519.431093:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766519.431096:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766519.431104:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741969024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766519.431111:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766519.431113:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766519.431114:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766519.431117:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2407 00000100:00080000:1.0:1775766519.431120:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766519.431122:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766519.431124:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766519.431132:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741969152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766519.431140:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766519.431175:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766519/real 1775766519] req@ffff90c203496d80 x1862023741968768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766519.431189:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741968768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766519.431197:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766519.431200:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766519.431203:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741968768:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766519.431220:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968896:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766519.431235:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766519.431243:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766519/real 1775766519] req@ffff90c203495880 x1862023741968896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766519.431251:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741968896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766519.431256:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766519.431258:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766519.431260:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741968896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766519.431264:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969024:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766519.431273:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766519.431279:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766519/real 1775766519] req@ffff90c203497100 x1862023741969024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766519.431287:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741969024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766519.431292:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766519.431294:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766519.431295:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766519.431299:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969152:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766519.431308:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766519.431313:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766519/real 1775766519] req@ffff90c203495c00 x1862023741969152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766519.431319:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741969152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766519.431326:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766519.431328:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766519.431330:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969152:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766520.455028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766521.479097:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766522.503003:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766523.527061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766524.551009:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766524.551018:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766524.551023:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766524.551035:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2413 00000100:00080000:1.0:1775766524.551048:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766524.551056:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766524.551068:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766524.551105:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741969280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766524.551133:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766524.551136:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766524.551137:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766524.551140:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2413 00000100:00080000:1.0:1775766524.551143:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766524.551146:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766524.551156:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766524.551165:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741969408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766524.551172:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741969280:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766524.551181:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766524.551184:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766524.551185:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766524.551189:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2413 00000100:00080000:1.0:1775766524.551191:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766524.551194:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766524.551198:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766524.551207:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741969536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766524.551212:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766524.551215:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766524.551216:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766524.551220:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2413 00000100:00080000:1.0:1775766524.551225:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766524.551229:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766524.551235:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766524.551250:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766524.551254:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741969664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766524.551308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766524/real 1775766524] req@ffff90c203494380 x1862023741969280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766524.551327:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741969280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766524.551336:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766524.551340:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766524.551343:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741969280:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766524.551362:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766524.551379:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766524.551388:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766524/real 1775766524] req@ffff90c203495f80 x1862023741969408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766524.551396:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741969408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766524.551401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766524.551404:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766524.551406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766524.551410:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969536:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766524.551420:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766524.551426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766524/real 1775766524] req@ffff90c203494000 x1862023741969536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766524.551433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741969536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766524.551438:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766524.551440:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766524.551441:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766524.551446:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969664:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766524.551456:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766524.551462:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766524/real 1775766524] req@ffff90c203497480 x1862023741969664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766524.551469:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741969664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766524.551476:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766524.551478:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766524.551479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969664:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766525.575054:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766526.599029:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766527.623097:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766528.647088:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766529.670968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766529.670987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766529.670992:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766529.671005:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2418 00000100:00080000:1.0:1775766529.671030:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766529.671037:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766529.671042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766529.671084:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741969792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766529.671110:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766529.671114:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766529.671115:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766529.671118:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2418 00000100:00080000:1.0:1775766529.671120:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766529.671123:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766529.671134:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766529.671145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741969920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766529.671156:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741969792:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766529.671162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766529.671168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766529.671169:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766529.671184:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2418 00000100:00080000:1.0:1775766529.671185:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766529.671188:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766529.671190:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766529.671200:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741970048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766529.671206:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766529.671208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766529.671209:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766529.671213:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2418 00000100:00080000:1.0:1775766529.671215:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766529.671218:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766529.671222:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766529.671229:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741970176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766529.671246:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766529.671284:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766529/real 1775766529] req@ffff90c203494e00 x1862023741969792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766529.671299:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741969792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766529.671307:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766529.671311:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766529.671315:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741969792:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766529.671334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766529.671349:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766529.671357:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766529/real 1775766529] req@ffff90c203497480 x1862023741969920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766529.671366:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741969920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766529.671372:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766529.671374:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766529.671376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741969920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766529.671380:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970048:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766529.671391:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766529.671397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766529/real 1775766529] req@ffff90c203494000 x1862023741970048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766529.671405:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741970048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766529.671410:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766529.671412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766529.671414:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766529.671417:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970176:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766529.671427:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766529.671432:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766529/real 1775766529] req@ffff90c203495f80 x1862023741970176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766554 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766529.671439:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741970176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766554 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766529.671451:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766529.671453:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766529.671455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970176:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766530.695064:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766531.719092:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766532.743134:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766533.767049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766534.790980:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766534.790989:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766534.790993:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766534.791001:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2423 00000100:00080000:1.0:1775766534.791011:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766534.791018:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766534.791043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766534.791053:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741970304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766534.791077:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766534.791079:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766534.791081:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766534.791083:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2423 00000100:00080000:1.0:1775766534.791086:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766534.791089:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766534.791097:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766534.791108:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741970432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766534.791109:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741970304:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766534.791129:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766534.791132:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766534.791133:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766534.791137:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2423 00000100:00080000:1.0:1775766534.791139:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766534.791141:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766534.791144:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766534.791153:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741970560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766534.791159:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766534.791162:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766534.791163:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766534.791165:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2423 00000100:00080000:1.0:1775766534.791169:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766534.791171:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766534.791175:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766534.791179:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766534.791186:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741970688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766534.791227:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766534/real 1775766534] req@ffff90c203494380 x1862023741970304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766534.791240:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741970304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766534.791248:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766534.791251:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766534.791261:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741970304:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766534.791279:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970432:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766534.791294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766534.791302:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766534/real 1775766534] req@ffff90c203495c00 x1862023741970432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766534.791310:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741970432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766534.791315:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766534.791317:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766534.791319:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766534.791324:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970560:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766534.791334:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766534.791340:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766534/real 1775766534] req@ffff90c203497100 x1862023741970560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766534.791347:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741970560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766534.791351:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766534.791353:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766534.791355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766534.791358:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970688:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766534.791368:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766534.791373:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766534/real 1775766534] req@ffff90c203495880 x1862023741970688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766559 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766534.791379:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741970688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766559 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766534.791385:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766534.791387:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766534.791389:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970688:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766535.815038:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766536.839024:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766537.863046:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766538.887062:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766539.911018:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766539.911035:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766539.911043:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766539.911060:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2428 00000100:00080000:1.0:1775766539.911078:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766539.911091:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766539.911131:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766539.911154:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741970816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766539.911207:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766539.911215:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766539.911219:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766539.911226:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2428 00000100:00080000:1.0:1775766539.911235:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766539.911244:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766539.911258:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766539.911279:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741970944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766539.911291:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741970816:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766539.911299:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766539.911308:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766539.911311:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766539.911317:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2428 00000100:00080000:1.0:1775766539.911322:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766539.911326:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766539.911332:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766539.911348:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741971072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766539.911361:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766539.911366:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766539.911368:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766539.911373:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2428 00000100:00080000:1.0:1775766539.911377:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766539.911381:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766539.911387:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766539.911399:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741971200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766539.911412:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766539.911514:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766539/real 1775766539] req@ffff90c203496d80 x1862023741970816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766539.911551:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741970816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766539.911565:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766539.911571:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766539.911576:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741970816:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766539.911607:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766539.911645:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766539.911668:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766539/real 1775766539] req@ffff90c203497b80 x1862023741970944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766539.911692:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741970944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766539.911709:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766539.911716:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766539.911722:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741970944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766539.911734:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766539.911800:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766539.911823:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766539/real 1775766539] req@ffff90c203496300 x1862023741971072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766539.911847:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741971072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766539.911917:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766539.911925:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766539.911931:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766539.911944:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971200:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766539.911986:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766539.912009:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766539/real 1775766539] req@ffff90c203496a00 x1862023741971200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766539.912035:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741971200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766564 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766539.912054:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766539.912061:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766539.912068:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971200:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766540.935057:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766541.959237:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766542.983078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766544.007050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766545.031049:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766545.031066:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766545.031074:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766545.031091:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2433 00010000:00010000:3.0:1775766545.031106:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766545.031109:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766545.031121:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766545.031175:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741971328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766545.031211:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766545.031215:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766545.031217:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766545.031221:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2433 00000100:00080000:1.0:1775766545.031225:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766545.031229:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766545.031241:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766545.031251:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741971328:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766545.031258:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741971456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766545.031278:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766545.031282:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766545.031284:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766545.031290:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2433 00000100:00080000:1.0:1775766545.031295:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766545.031299:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766545.031304:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766545.031316:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741971584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766545.031324:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766545.031327:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766545.031329:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766545.031333:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2433 00000800:00000100:2.0:1775766545.031345:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766545.031347:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766545.031353:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766545.031386:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766545.031404:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741971712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766545.031414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766545/real 1775766545] req@ffff90c203496680 x1862023741971328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766545.031433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741971328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766545.031441:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766545.031445:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766545.031448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741971328:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766545.031476:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766545.031502:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766545.031515:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766545/real 1775766545] req@ffff90c203494a80 x1862023741971456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766545.031529:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741971456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766545.031538:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766545.031542:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766545.031545:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766545.031550:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971584:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766545.031566:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766545.031573:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766545/real 1775766545] req@ffff90c203495180 x1862023741971584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766545.031581:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741971584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766545.031586:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766545.031588:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766545.031590:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766545.031594:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971712:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766545.031605:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766545.031610:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766545/real 1775766545] req@ffff90c203495500 x1862023741971712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766545.031618:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741971712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766545.031624:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766545.031626:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766545.031628:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971712:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766546.055089:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766547.079029:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766548.103111:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766549.127222:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766550.151038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766550.151048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766550.151053:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766550.151062:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2438 00000100:00080000:0.0:1775766550.151075:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766550.151098:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766550.151151:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741971840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766550.151173:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766550.151194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766550.151199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766550.151202:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766550.151206:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2438 00000100:00080000:0.0:1775766550.151212:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766550.151218:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766550.151233:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766550.151239:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741971840:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766550.151259:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741971968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766550.151278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766550.151285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766550.151288:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766550.151309:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2438 00000100:00080000:0.0:1775766550.151340:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766550.151345:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766550.151350:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766550.151356:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766550.151362:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741972096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766550.151371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766550.151374:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766550.151376:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766550.151379:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2438 00000100:00080000:0.0:1775766550.151382:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766550.151384:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766550.151389:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766550.151397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766550/real 1775766550] req@ffff90c20363aa00 x1862023741971840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766550.151406:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741972224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766550.151416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741971840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766550.151425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766550.151430:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766550.151433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741971840:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766550.151453:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766550.151469:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766550.151477:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766550/real 1775766550] req@ffff90c203638000 x1862023741971968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766550.151484:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741971968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766550.151495:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766550.151497:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766550.151499:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741971968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766550.151503:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766550.151512:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766550.151518:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766550/real 1775766550] req@ffff90c203639180 x1862023741972096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766550.151525:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741972096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766550.151531:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766550.151532:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766550.151534:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766550.151538:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972224:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766550.151547:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766550.151552:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766550/real 1775766550] req@ffff90c203639500 x1862023741972224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766550.151559:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741972224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766550.151565:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766550.151567:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766550.151570:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972224:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766551.175031:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766552.199091:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766553.223039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766554.247157:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766555.271082:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766555.271093:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766555.271100:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:0.0:1775766555.271107:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766555.271114:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2443 00000100:00080000:3.0:1775766555.271129:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766555.271141:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766555.271218:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741972352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766555.271298:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766555.271321:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766555.271324:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766555.271331:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2443 00000100:00080000:3.0:1775766555.271337:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766555.271342:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766555.271355:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766555.271375:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741972480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766555.271390:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741972352:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766555.271394:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766555.271407:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766555.271410:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766555.271417:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2443 00000100:00080000:3.0:1775766555.271420:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766555.271424:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766555.271431:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766555.271446:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741972608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766555.271458:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766555.271462:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766555.271464:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766555.271468:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2443 00000100:00080000:3.0:1775766555.271494:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766555.271503:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766555.271507:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775766555.271513:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766555.271549:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741972736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766555.271601:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766555/real 1775766555] req@ffff90c218da1880 x1862023741972352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766555.271631:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741972352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766555.271647:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766555.271653:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766555.271659:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741972352:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766555.271689:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766555.271718:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766555.271733:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766555/real 1775766555] req@ffff90c218da3800 x1862023741972480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766555.271749:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741972480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766555.271759:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766555.271763:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766555.271767:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766555.271774:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766555.271792:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766555.271803:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766555/real 1775766555] req@ffff90c218da2d80 x1862023741972608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766555.271817:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741972608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766555.271827:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766555.271830:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766555.271833:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766555.271840:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972736:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766555.271857:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766555.271867:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766555/real 1775766555] req@ffff90c218da0e00 x1862023741972736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766555.271880:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741972736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766555.271891:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766555.271894:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766555.271898:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972736:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766556.295168:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766557.319108:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766558.343074:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766559.367068:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766560.391044:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766560.391059:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766560.391068:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766560.391085:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2448 00000100:00080000:3.0:1775766560.391130:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766560.391140:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766560.391277:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741972864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766560.391315:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:1.0:1775766560.391319:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766560.391320:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766560.391326:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766560.391332:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2448 00000100:00080000:3.0:1775766560.391337:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766560.391344:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766560.391358:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766560.391377:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741972992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766560.391390:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766560.391397:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766560.391403:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766560.391409:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741972864:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766560.391410:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2448 00000100:00080000:3.0:1775766560.391421:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766560.391435:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766560.391441:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766560.391451:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741973120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766560.391461:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766560.391464:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766560.391467:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766560.391472:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2448 00000100:00080000:3.0:1775766560.391475:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766560.391479:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766560.391483:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766560.391496:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741973248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766560.391631:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766560.391692:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766560.391729:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973248:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766561.415166:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766562.439105:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766563.463141:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775766563.463343:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775766563.463407:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775766563.463434:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766563.463491:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766563.463520:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766563.463539:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766563.463566:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766560/real 1775766563] req@ffff90c218da3480 x1862023741972992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766563.463607:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741972992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766563.463625:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766563.463633:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766563.463639:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741972992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766563.463652:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766560/real 1775766563] req@ffff90c218da0700 x1862023741973120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766563.463670:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741973120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766563.463681:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766563.463687:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766563.463691:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766563.463701:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766560/real 1775766563] req@ffff90c218da0380 x1862023741973248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766563.463718:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741973248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766563.463734:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766563.463740:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766563.463745:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973248:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766563.463757:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766560/real 1775766563] req@ffff90c218da0a80 x1862023741972864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766563.463777:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741972864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766563.463809:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766563.463814:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766563.463820:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741972864:192.168.202.155@tcp:250:kworker.0 00010000:00010000:0.0:1775766564.487051:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766565.511002:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766565.511011:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766565.511016:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766565.511027:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2453 00000100:00080000:3.0:1775766565.511039:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766565.511047:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766565.511090:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741973376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775766565.511120:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766565.511121:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766565.511127:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766565.511130:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766565.511136:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2453 00000100:00080000:3.0:1775766565.511142:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766565.511146:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766565.511189:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766565.511214:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741973504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766565.511215:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741973376:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766565.511265:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766565.511270:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766565.511272:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766565.511281:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2453 00000100:00080000:3.0:1775766565.511286:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766565.511290:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766565.511296:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766565.511311:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741973632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766565.511320:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766565.511323:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766565.511324:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766565.511328:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2453 00000100:00080000:3.0:1775766565.511350:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766565.511354:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766565.511360:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766565.511386:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766565.511402:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741973760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766565.511653:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766565/real 1775766565] req@ffff90c218da3100 x1862023741973376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766565.511694:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741973376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766565.511718:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766565.511723:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766565.511729:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741973376:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766565.511767:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766565.511833:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766565.511858:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766565/real 1775766565] req@ffff90c218da0000 x1862023741973504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766565.511907:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741973504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766565.511917:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766565.511920:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766565.511923:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766565.511931:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766565.511955:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766565.511969:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766565/real 1775766565] req@ffff90c218da1f80 x1862023741973632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766565.511981:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741973632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766565.511989:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766565.511992:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766565.511995:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766565.512001:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973760:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766565.512019:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766565.512051:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766565/real 1775766565] req@ffff90c218da0a80 x1862023741973760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766565.512075:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741973760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766565.512086:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766565.512090:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766565.512094:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741973760:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766566.535250:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766567.559254:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766568.583058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766569.607017:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766570.630939:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766570.630947:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766570.630952:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766570.630961:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2459 00000100:00080000:3.0:1775766570.630971:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766570.630977:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766570.631010:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741973888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775766570.631033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766570.631042:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766570.631045:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766570.631047:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766570.631050:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2459 00000100:00080000:3.0:1775766570.631052:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766570.631055:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766570.631062:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766570.631071:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741974016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766570.631080:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766570.631085:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741973888:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766570.631085:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766570.631086:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766570.631094:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2459 00000100:00080000:3.0:1775766570.631097:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766570.631099:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766570.631115:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766570.631126:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741974144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766570.631135:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766570.631138:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766570.631139:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766570.631142:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2459 00000100:00080000:3.0:1775766570.631166:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766570.631169:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766570.631173:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766570.631177:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766570.631186:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741974272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766570.631214:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766570/real 1775766570] req@ffff90c218da0380 x1862023741973888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766570.631229:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741973888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766570.631237:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766570.631240:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766570.631243:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741973888:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766570.631262:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766570.631277:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766570.631284:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766570/real 1775766570] req@ffff90c218da0700 x1862023741974016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766570.631292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741974016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766570.631297:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766570.631299:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766570.631301:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766570.631305:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766570.631314:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766570.631321:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766570/real 1775766570] req@ffff90c218da3480 x1862023741974144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766570.631328:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741974144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766570.631332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766570.631334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766570.631336:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766570.631340:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974272:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766570.631349:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766570.631354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766570/real 1775766570] req@ffff90c218da0e00 x1862023741974272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766595 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766570.631361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741974272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766595 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766570.631367:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766570.631369:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766570.631371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974272:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766571.655064:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766572.679129:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766573.703024:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766574.727054:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766575.750986:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766575.751008:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766575.751017:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766575.751028:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2464 00000100:00080000:3.0:1775766575.751040:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766575.751046:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766575.751076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766575.751095:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741974400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766575.751119:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766575.751122:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766575.751124:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766575.751126:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2464 00000100:00080000:3.0:1775766575.751129:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766575.751132:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766575.751140:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766575.751148:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741974528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766575.751153:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741974400:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766575.751161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766575.751164:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766575.751165:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766575.751168:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2464 00000100:00080000:3.0:1775766575.751170:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766575.751172:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766575.751175:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766575.751182:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741974656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766575.751188:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766575.751190:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766575.751191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766575.751195:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2464 00000100:00080000:3.0:1775766575.751207:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766575.751210:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766575.751217:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775766575.751218:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766575.751230:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741974784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766575.751251:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766575/real 1775766575] req@ffff90c218da2d80 x1862023741974400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766575.751265:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741974400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766575.751274:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766575.751277:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766575.751280:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741974400:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766575.751297:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974528:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766575.751311:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766575.751318:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766575/real 1775766575] req@ffff90c218da3800 x1862023741974528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766575.751326:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741974528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766575.751331:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766575.751333:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766575.751335:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766575.751338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766575.751348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766575.751353:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766575/real 1775766575] req@ffff90c218da1880 x1862023741974656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766575.751360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741974656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766575.751365:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766575.751367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766575.751369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766575.751373:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974784:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766575.751382:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766575.751387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766575/real 1775766575] req@ffff90c218da0e00 x1862023741974784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766600 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766575.751394:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741974784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766600 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766575.751400:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766575.751402:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766575.751404:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741974784:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766576.775004:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766577.799210:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766578.822988:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766579.847147:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766580.871097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766580.871110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766580.871117:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766580.871130:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2469 00000100:00080000:3.0:1775766580.871145:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766580.871154:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766580.871209:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766580.871212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741974912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766580.871247:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766580.871252:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766580.871254:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766580.871259:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2469 00000100:00080000:3.0:1775766580.871262:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766580.871267:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766580.871279:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766580.871301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741974912:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766580.871311:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741975040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766580.871324:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766580.871329:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766580.871331:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766580.871336:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2469 00000100:00080000:3.0:1775766580.871340:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766580.871343:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766580.871349:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766580.871361:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741975168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766580.871371:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766580.871375:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766580.871377:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766580.871381:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2469 00000100:00080000:3.0:1775766580.871384:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766580.871387:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766580.871393:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766580.871399:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766580.871411:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741975296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766580.871458:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766580/real 1775766580] req@ffff90c218da3480 x1862023741974912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766580.871481:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741974912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766580.871494:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766580.871499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766580.871504:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741974912:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766580.871534:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766580.871557:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766580.871569:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766580/real 1775766580] req@ffff90c218da0700 x1862023741975040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766580.871582:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741975040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766580.871590:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766580.871594:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766580.871597:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766580.871602:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766580.871617:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766580.871625:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766580/real 1775766580] req@ffff90c218da0380 x1862023741975168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766580.871636:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741975168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766580.871643:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766580.871646:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766580.871649:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766580.871654:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975296:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766580.871668:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766580.871676:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766580/real 1775766580] req@ffff90c218da0a80 x1862023741975296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766580.871686:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741975296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766605 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766580.871703:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766580.871706:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766580.871710:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975296:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766581.895209:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766582.919154:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766583.943186:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766584.967088:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766585.991019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766585.991031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766585.991036:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766585.991047:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2474 00010000:00010000:0.0:1775766585.991067:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766585.991071:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766585.991081:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766585.991121:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741975424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766585.991152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766585.991157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766585.991159:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766585.991165:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2474 00000100:00080000:3.0:1775766585.991169:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766585.991174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766585.991186:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766585.991190:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741975424:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766585.991203:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741975552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766585.991217:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766585.991221:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766585.991224:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766585.991228:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2474 00000100:00080000:3.0:1775766585.991267:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766585.991283:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766585.991290:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775766585.991294:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766585.991313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741975680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766585.991327:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766585.991332:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766585/real 1775766585] req@ffff90c218da1f80 x1862023741975424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766585.991333:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766585.991338:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766585.991351:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2474 00000100:00080000:3.0:1775766585.991358:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766585.991361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741975424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766585.991370:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775766585.991375:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775766585.991379:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775766585.991381:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766585.991387:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741975424:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766585.991405:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741975808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766585.991412:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975552:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766585.991453:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766585.991462:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766585/real 1775766585] req@ffff90c218da0000 x1862023741975552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766585.991471:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741975552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766585.991477:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766585.991479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766585.991481:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766585.991486:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766585.991496:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766585.991505:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766585/real 1775766585] req@ffff90c218da3100 x1862023741975680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766585.991517:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741975680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766585.991525:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766585.991528:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766585.991532:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766585.991546:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975808:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766585.991565:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766585.991573:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766585/real 1775766585] req@ffff90c218da0a80 x1862023741975808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766585.991581:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741975808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766610 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766585.991590:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766585.991594:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766585.991597:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741975808:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766587.015037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766588.039033:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766589.063067:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766590.087051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766591.111062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766591.111074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766591.111079:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766591.111093:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2479 00000100:00080000:1.0:1775766591.111108:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766591.111129:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766591.111131:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766591.111175:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741975936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766591.111203:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766591.111206:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766591.111208:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766591.111212:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2479 00000100:00080000:1.0:1775766591.111216:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766591.111221:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766591.111231:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766591.111246:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741975936:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766591.111247:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741976064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766591.111263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766591.111266:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766591.111267:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766591.111270:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2479 00000100:00080000:1.0:1775766591.111274:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766591.111277:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766591.111281:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766591.111296:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741976192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766591.111309:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766591.111313:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766591.111315:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766591.111321:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2479 00000800:00000100:2.0:1775766591.111325:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766591.111332:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766591.111340:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766591.111348:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766591.111365:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741976320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766591.111379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766591/real 1775766591] req@ffff90c203494700 x1862023741975936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766591.111418:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741975936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766591.111431:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766591.111435:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766591.111438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741975936:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766591.111461:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766591.111521:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766591.111536:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766591/real 1775766591] req@ffff90c203495500 x1862023741976064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766591.111572:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741976064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766591.111580:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766591.111582:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766591.111585:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766591.111589:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766591.111606:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766591.111619:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766591/real 1775766591] req@ffff90c203495180 x1862023741976192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766591.111634:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741976192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766591.111674:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766591.111681:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766591.111688:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766591.111699:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976320:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766591.111746:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766591.111759:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766591/real 1775766591] req@ffff90c203494a80 x1862023741976320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766591.111771:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741976320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766591.111782:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766591.111786:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766591.111789:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976320:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766592.135016:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766593.159056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766594.183110:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766595.207072:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766596.230974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766596.230989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766596.230997:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766596.231010:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2484 00000100:00080000:1.0:1775766596.231025:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775766596.231035:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766596.231035:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766596.231086:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741976448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766596.231117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766596.231121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766596.231123:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766596.231128:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2484 00000100:00080000:1.0:1775766596.231132:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766596.231137:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766596.231149:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766596.231154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741976448:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766596.231168:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741976576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766596.231180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766596.231184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766596.231186:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766596.231190:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2484 00000100:00080000:1.0:1775766596.231194:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766596.231199:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766596.231206:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766596.231219:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741976704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766596.231228:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766596.231234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766596.231238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766596.231240:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766596.231247:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2484 00000100:00080000:1.0:1775766596.231251:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766596.231256:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766596.231263:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766596.231270:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766596/real 1775766596] req@ffff90c203496680 x1862023741976448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766596.231279:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741976832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766596.231292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741976448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766596.231301:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766596.231306:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766596.231310:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741976448:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766596.231334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766596.231348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766596.231356:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766596/real 1775766596] req@ffff90c203496a00 x1862023741976576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766596.231365:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741976576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766596.231370:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766596.231372:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766596.231375:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766596.231378:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766596.231388:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766596.231393:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766596/real 1775766596] req@ffff90c203496300 x1862023741976704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766596.231400:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741976704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766596.231405:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766596.231407:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766596.231408:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766596.231412:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976832:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766596.231422:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766596.231427:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766596/real 1775766596] req@ffff90c203497b80 x1862023741976832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766596.231434:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741976832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766596.231442:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766596.231443:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766596.231445:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741976832:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766597.255098:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766598.279062:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766599.303114:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766600.327152:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766601.351014:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766601.351027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766601.351032:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766601.351054:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2489 00000100:00080000:1.0:1775766601.351066:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766601.351073:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766601.351098:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766601.351122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741976960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766601.351148:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766601.351151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766601.351152:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766601.351155:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2489 00000100:00080000:1.0:1775766601.351159:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766601.351163:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766601.351172:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766601.351185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741977088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766601.351185:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741976960:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766601.351201:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766601.351211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766601.351212:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766601.351216:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2489 00000100:00080000:1.0:1775766601.351219:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766601.351221:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766601.351226:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766601.351235:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741977216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766601.351243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766601.351246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766601.351247:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766601.351250:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2489 00000100:00080000:1.0:1775766601.351253:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766601.351258:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766601.351260:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766601.351265:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766601.351273:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741977344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766601.351299:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766601/real 1775766601] req@ffff90c203496d80 x1862023741976960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766601.351317:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741976960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766601.351325:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766601.351329:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766601.351332:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741976960:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766601.351352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977088:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766601.351382:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766601.351403:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766601/real 1775766601] req@ffff90c203495880 x1862023741977088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766601.351412:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741977088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766601.351423:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766601.351425:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766601.351428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766601.351432:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977216:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766601.351443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766601.351449:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766601/real 1775766601] req@ffff90c203497100 x1862023741977216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766601.351457:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741977216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766601.351465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766601.351469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766601.351474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766601.351481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977344:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766601.351502:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766601.351515:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766601/real 1775766601] req@ffff90c203495c00 x1862023741977344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766601.351530:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741977344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766601.351542:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766601.351544:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766601.351547:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977344:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766602.375097:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766603.399062:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766604.423109:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766605.447001:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766606.471040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766606.471052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766606.471059:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766606.471072:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2494 00010000:00010000:1.0:1775766606.471082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766606.471089:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766606.471099:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766606.471160:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741977472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766606.471213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766606.471217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766606.471219:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766606.471223:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2494 00000100:00080000:3.0:1775766606.471229:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766606.471233:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766606.471244:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766606.471271:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741977472:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766606.471282:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741977600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766606.471295:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766606.471299:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766606.471301:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766606.471305:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2494 00000100:00080000:3.0:1775766606.471309:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766606.471313:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766606.471318:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766606.471329:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741977728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766606.471339:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766606.471342:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766606.471343:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766606.471350:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2494 00000800:00000100:2.0:1775766606.471354:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766606.471357:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766606.471362:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766606.471367:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766606.471381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741977856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766606.471432:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766606/real 1775766606] req@ffff90c218da0380 x1862023741977472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766606.471476:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741977472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766606.471490:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766606.471496:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766606.471502:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741977472:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766606.471534:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766606.471567:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766606.471584:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766606/real 1775766606] req@ffff90c218da0700 x1862023741977600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766606.471603:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741977600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766606.471621:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766606.471624:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766606.471628:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766606.471634:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766606.471655:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766606.471665:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766606/real 1775766606] req@ffff90c218da3480 x1862023741977728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766606.471676:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741977728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766606.471683:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766606.471685:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766606.471688:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766606.471693:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977856:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766606.471706:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766606.471713:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766606/real 1775766606] req@ffff90c218da0e00 x1862023741977856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766606.471768:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741977856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766606.471791:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766606.471796:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766606.471801:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741977856:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766607.495043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766608.519047:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766609.543047:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766610.567047:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766611.590942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766611.590951:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766611.590956:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766611.590965:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2500 00000100:00080000:3.0:1775766611.590976:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766611.590983:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766611.591017:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741977984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766611.591040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766611.591043:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766611.591045:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766611.591047:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2500 00000100:00080000:3.0:1775766611.591049:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766611.591052:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766611.591059:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766611.591069:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741978112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766611.591080:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741977984:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766611.591082:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766611.591088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766611.591090:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766611.591094:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2500 00000100:00080000:3.0:1775766611.591096:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766611.591100:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766611.591107:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00010000:00010000:0.0:1775766611.591111:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766611.591120:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741978240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766611.591137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766611.591141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766611.591143:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766611.591148:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2500 00000100:00080000:3.0:1775766611.591152:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766611.591158:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766611.591158:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766611.591167:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766611.591183:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741978368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766611.591195:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766611/real 1775766611] req@ffff90c218da1880 x1862023741977984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766611.591212:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741977984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766611.591221:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766611.591224:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766611.591227:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741977984:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766611.591245:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766611.591260:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766611.591268:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766611/real 1775766611] req@ffff90c218da3800 x1862023741978112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766611.591275:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741978112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766611.591281:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766611.591283:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766611.591285:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766611.591289:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978240:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766611.591298:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766611.591303:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766611/real 1775766611] req@ffff90c218da2d80 x1862023741978240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766611.591310:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741978240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766611.591315:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766611.591317:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766611.591319:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766611.591323:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978368:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766611.591333:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766611.591337:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766611/real 1775766611] req@ffff90c218da0e00 x1862023741978368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766611.591344:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741978368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766611.591350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766611.591352:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766611.591354:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978368:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766612.615056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766613.639028:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766614.663053:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766615.687058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766616.710962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766616.710973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766616.710977:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766616.710988:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2505 00000100:00080000:3.0:1775766616.711007:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766616.711014:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766616.711035:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766616.711054:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741978496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766616.711084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766616.711089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766616.711091:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766616.711096:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2505 00000100:00080000:3.0:1775766616.711100:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766616.711105:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766616.711114:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741978496:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775766616.711115:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766616.711134:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741978624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766616.711147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766616.711151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766616.711153:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766616.711157:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2505 00000100:00080000:3.0:1775766616.711170:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766616.711175:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766616.711176:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766616.711183:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766616.711196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741978752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766616.711206:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766616.711210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1775766616.711213:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766616/real 1775766616] req@ffff90c218da3480 x1862023741978496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775766616.711214:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766616.711225:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2505 00000100:00080000:3.0:1775766616.711230:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766616.711235:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766616.711238:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741978496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775766616.711243:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775766616.711249:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766616.711254:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766616.711258:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741978496:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766616.711261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741978880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766616.711281:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766616.711315:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766616.711329:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766616/real 1775766616] req@ffff90c218da0700 x1862023741978624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766616.711343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741978624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766616.711352:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766616.711356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766616.711359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766616.711363:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766616.711378:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766616.711386:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766616/real 1775766616] req@ffff90c218da0380 x1862023741978752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766616.711393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741978752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766616.711398:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766616.711401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766616.711403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766616.711413:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978880:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766616.711423:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766616.711428:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766616/real 1775766616] req@ffff90c218da0a80 x1862023741978880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766641 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766616.711435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741978880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766641 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766616.711446:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766616.711448:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766616.711451:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741978880:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766617.734995:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766618.759027:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766619.783168:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766620.807060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766621.830989:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766621.830998:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766621.831002:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766621.831010:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2510 00000100:00080000:1.0:1775766621.831021:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766621.831025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766621.831028:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766621.831063:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741979008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766621.831085:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766621.831088:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766621.831089:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766621.831092:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2510 00000100:00080000:1.0:1775766621.831094:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766621.831096:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766621.831104:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766621.831114:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741979136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766621.831118:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741979008:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766621.831126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766621.831129:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766621.831130:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766621.831134:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2510 00000100:00080000:1.0:1775766621.831136:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766621.831139:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766621.831142:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766621.831149:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741979264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766621.831156:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766621.831158:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766621.831159:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766621.831161:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2510 00000100:00080000:1.0:1775766621.831163:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766621.831166:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766621.831169:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766621.831176:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741979392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766621.831183:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766621.831228:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766621/real 1775766621] req@ffff90c203494380 x1862023741979008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766621.831242:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741979008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766621.831252:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766621.831256:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766621.831260:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741979008:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766621.831283:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766621.831307:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766621.831319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766621/real 1775766621] req@ffff90c203495f80 x1862023741979136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766621.831328:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741979136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766621.831334:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766621.831336:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766621.831338:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766621.831342:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766621.831358:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766621.831368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766621/real 1775766621] req@ffff90c203494000 x1862023741979264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766621.831380:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741979264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766621.831389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766621.831393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766621.831396:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766621.831403:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979392:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766621.831421:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766621.831428:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766621/real 1775766621] req@ffff90c203497480 x1862023741979392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766621.831436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741979392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766646 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766621.831443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766621.831445:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766621.831447:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979392:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766622.855045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766623.879070:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766624.903087:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766625.927057:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766626.950990:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766626.951000:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766626.951005:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766626.951013:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2515 00000100:00080000:1.0:1775766626.951024:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766626.951030:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766626.951060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766626.951067:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741979520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766626.951090:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766626.951093:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766626.951095:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766626.951098:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2515 00000100:00080000:1.0:1775766626.951100:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766626.951104:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766626.951112:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766626.951119:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741979520:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766626.951125:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741979648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766626.951134:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766626.951136:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766626.951138:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766626.951141:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2515 00000100:00080000:1.0:1775766626.951144:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766626.951145:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766626.951149:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766626.951156:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741979776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766626.951163:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766626.951165:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766626.951167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766626.951169:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2515 00000100:00080000:1.0:1775766626.951172:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766626.951174:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766626.951177:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766626.951185:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741979904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766626.951213:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766626.951238:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766626.951254:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979904:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766627.975085:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766628.999060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766630.023193:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775766630.023213:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775766630.023268:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775766630.023292:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766630.023335:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766630.023346:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775766630.023357:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766630.023387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766626/real 1775766630] req@ffff90c203497480 x1862023741979648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766630.023419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741979648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766630.023435:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766630.023442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766630.023447:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766630.023457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766626/real 1775766630] req@ffff90c203494000 x1862023741979776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766630.023478:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741979776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766630.023502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766630.023507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766630.023511:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766630.023520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766626/real 1775766630] req@ffff90c203495f80 x1862023741979904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766630.023533:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741979904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766630.023553:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766630.023557:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766630.023561:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741979904:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766630.023569:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766626/real 1775766630] req@ffff90c203494e00 x1862023741979520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766630.023582:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741979520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766651 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766630.023596:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766630.023605:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766630.023608:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741979520:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766631.047006:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766632.071048:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766632.071063:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766632.071071:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766632.071086:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2520 00000100:00080000:1.0:1775766632.071106:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766632.071121:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766632.071185:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766632.071206:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741980032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766632.071283:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766632.071289:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766632.071292:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766632.071300:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2520 00000100:00080000:1.0:1775766632.071318:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766632.071323:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766632.071342:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766632.071352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741980032:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766632.071399:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741980160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766632.071425:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766632.071431:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766632.071437:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766632.071448:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2520 00000800:00000100:2.0:1775766632.071450:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766632.071489:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766632.071499:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766632.071510:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766632.071529:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766632/real 1775766632] req@ffff90c203494380 x1862023741980032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766632.071540:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741980288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766632.071581:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766632.071583:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741980032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766632.071603:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766632.071612:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766632.071617:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766632.071629:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00080000:1.0:1775766632.071630:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2520 00000100:00100000:2.0:1775766632.071639:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741980032:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766632.071657:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766632.071671:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766632.071679:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980160:192.168.202.155@tcp:38:kworker.0 00000008:00100000:1.0:1775766632.071687:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766632.071719:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741980416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766632.071739:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766632.071778:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766632/real 1775766632] req@ffff90c203495c00 x1862023741980160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766632.071802:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741980160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766632.071812:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766632.071816:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766632.071821:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766632.071831:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766632.071866:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766632.071918:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766632/real 1775766632] req@ffff90c203497100 x1862023741980288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766632.071933:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741980288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766632.071942:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766632.071945:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766632.071948:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766632.071969:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980416:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766632.072007:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766632.072019:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766632/real 1775766632] req@ffff90c203495880 x1862023741980416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766632.072040:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741980416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766632.072051:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766632.072055:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766632.072058:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980416:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766633.095134:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766634.119088:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766635.143035:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766636.167034:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766637.191014:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766637.191017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766637.191026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766637.191032:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766637.191043:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2525 00000100:00080000:3.0:1775766637.191055:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766637.191061:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766637.191105:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741980544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766637.191133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766637.191137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766637.191138:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766637.191141:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2525 00000100:00080000:3.0:1775766637.191143:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766637.191147:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766637.191156:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766637.191165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741980672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766637.191171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766637.191174:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766637.191175:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766637.191177:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2525 00000100:00080000:3.0:1775766637.191179:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766637.191181:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766637.191184:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766637.191190:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741980800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766637.191196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766637.191198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766637.191200:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766637.191203:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2525 00000100:00080000:3.0:1775766637.191206:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766637.191208:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766637.191211:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766637.191221:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741980544:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766637.191221:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741980928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766637.191303:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766637.191343:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766637/real 1775766637] req@ffff90c218da3100 x1862023741980544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766637.191357:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741980544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766637.191366:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766637.191370:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766637.191373:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741980544:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766637.191379:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766637.191393:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766637.191401:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766637/real 1775766637] req@ffff90c218da0000 x1862023741980672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766637.191427:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741980672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766637.191438:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766637.191441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766637.191443:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766637.191447:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766637.191462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766637.191469:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766637/real 1775766637] req@ffff90c218da1f80 x1862023741980800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766637.191476:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741980800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766637.191481:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766637.191483:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766637.191485:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766637.191509:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980928:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766637.191520:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766637.191526:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766637/real 1775766637] req@ffff90c218da0a80 x1862023741980928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766637.191533:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741980928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766637.191540:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766637.191542:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766637.191545:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741980928:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766638.215078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766639.238974:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766640.263025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766641.287005:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766642.310998:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766642.311010:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766642.311015:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766642.311027:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2530 00000100:00080000:3.0:1775766642.311039:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766642.311048:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766642.311105:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741981056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766642.311133:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766642.311138:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00010000:00010000:0.0:1775766642.311139:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766642.311140:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766642.311149:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2530 00000100:00080000:3.0:1775766642.311153:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766642.311173:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766642.311184:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766642.311204:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741981056:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766642.311212:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741981184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766642.311224:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766642.311227:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766642.311228:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766642.311232:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2530 00000100:00080000:3.0:1775766642.311236:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766642.311239:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766642.311243:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766642.311256:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741981312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766642.311265:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766642.311268:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766642.311269:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766642.311272:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2530 00000100:00080000:3.0:1775766642.311278:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766642.311282:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766642.311285:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766642.311294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766642.311299:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741981440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766642.311346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766642/real 1775766642] req@ffff90c218da0380 x1862023741981056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766642.311401:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741981056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766642.311428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766642.311432:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766642.311436:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741981056:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766642.311462:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981184:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766642.311483:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766642.311493:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766642/real 1775766642] req@ffff90c218da0700 x1862023741981184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766642.311503:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741981184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766642.311511:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766642.311515:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766642.311519:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766642.311526:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766642.311549:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766642.311563:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766642/real 1775766642] req@ffff90c218da3480 x1862023741981312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766642.311579:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741981312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766642.311600:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766642.311603:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766642.311606:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766642.311611:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981440:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766642.311629:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766642.311638:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766642/real 1775766642] req@ffff90c218da0e00 x1862023741981440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766642.311647:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741981440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766642.311655:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766642.311658:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766642.311660:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981440:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766643.335076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766644.359086:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766645.383075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766646.407017:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766647.430989:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766647.430997:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766647.431002:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766647.431029:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2535 00000100:00080000:3.0:1775766647.431041:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766647.431050:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766647.431093:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741981568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775766647.431096:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766647.431124:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766647.431128:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766647.431130:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766647.431149:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2535 00000100:00080000:3.0:1775766647.431152:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766647.431155:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766647.431165:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766647.431175:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741981696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766647.431185:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766647.431189:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741981568:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766647.431189:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766647.431190:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766647.431214:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2535 00000100:00080000:3.0:1775766647.431218:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766647.431223:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766647.431229:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766647.431243:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741981824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766647.431256:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766647.431261:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766647.431264:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766647.431270:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2535 00000100:00080000:3.0:1775766647.431295:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766647.431326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766647.431365:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766647.431372:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766647.431389:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741981952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766647.431398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766647/real 1775766647] req@ffff90c218da2d80 x1862023741981568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766647.431453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741981568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766647.431484:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766647.431490:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766647.431495:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741981568:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766647.431536:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766647.431568:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766647.431585:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766647/real 1775766647] req@ffff90c218da3800 x1862023741981696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766647.431602:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741981696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766647.431612:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766647.431616:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766647.431619:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766647.431640:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766647.431685:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766647.431698:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766647/real 1775766647] req@ffff90c218da1880 x1862023741981824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766647.431710:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741981824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766647.431719:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766647.431722:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766647.431726:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766647.431734:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981952:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766647.431757:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766647.431796:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766647/real 1775766647] req@ffff90c218da0e00 x1862023741981952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766647.431824:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741981952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766647.431838:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766647.431857:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766647.431861:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741981952:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766648.455084:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766649.479144:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766650.503094:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766651.527126:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766652.550980:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766652.550991:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766652.550996:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766652.551007:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2541 00000100:00080000:3.0:1775766652.551018:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766652.551024:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766652.551050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766652.551059:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741982080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766652.551083:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766652.551086:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766652.551088:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766652.551090:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2541 00000100:00080000:3.0:1775766652.551093:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766652.551096:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766652.551104:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766652.551112:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741982208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766652.551117:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741982080:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775766652.551123:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766652.551128:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766652.551130:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766652.551133:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2541 00000100:00080000:3.0:1775766652.551134:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766652.551137:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766652.551141:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766652.551149:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741982336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766652.551156:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766652.551158:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766652.551159:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766652.551162:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2541 00000100:00080000:3.0:1775766652.551183:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766652.551186:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766652.551188:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766652.551194:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766652.551198:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741982464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766652.551240:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766652/real 1775766652] req@ffff90c218da3480 x1862023741982080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766652.551279:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741982080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766652.551298:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766652.551301:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766652.551304:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741982080:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766652.551324:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982208:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766652.551342:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766652.551351:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766652/real 1775766652] req@ffff90c218da0700 x1862023741982208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766652.551360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741982208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766652.551373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766652.551375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766652.551378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766652.551382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982336:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766652.551393:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766652.551399:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766652/real 1775766652] req@ffff90c218da0380 x1862023741982336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766652.551407:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741982336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766652.551412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766652.551414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766652.551415:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766652.551419:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982464:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766652.551429:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766652.551434:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766652/real 1775766652] req@ffff90c218da0a80 x1862023741982464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766652.551441:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741982464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766652.551452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766652.551454:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766652.551456:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982464:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766653.575054:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766654.599054:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766655.623056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766656.647009:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766657.670983:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766657.670993:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766657.670998:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766657.671009:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2546 00000100:00080000:3.0:1775766657.671022:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766657.671030:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766657.671050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775766657.671072:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023741982592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766657.671096:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766657.671100:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766657.671114:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766657.671119:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2546 00000100:00080000:3.0:1775766657.671124:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766657.671128:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766657.671139:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766657.671151:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741982592:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766657.671152:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023741982720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766657.671164:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766657.671167:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766657.671169:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766657.671173:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2546 00000100:00080000:3.0:1775766657.671185:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766657.671203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766657.671208:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766657.671220:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023741982848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766657.671230:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766657.671233:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1775766657.671234:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775766657.671235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766657.671240:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2546 00000100:00080000:3.0:1775766657.671243:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766657.671246:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766657.671250:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775766657.671261:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023741982976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766657.671274:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766657/real 1775766657] req@ffff90c218da1f80 x1862023741982592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766657.671294:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023741982592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766657.671307:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766657.671311:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766657.671316:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741982592:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766657.671337:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766657.671365:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766657.671375:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766657/real 1775766657] req@ffff90c218da0000 x1862023741982720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766657.671384:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023741982720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766657.671390:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766657.671393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766657.671395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766657.671399:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766657.671411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766657.671417:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766657/real 1775766657] req@ffff90c218da3100 x1862023741982848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766657.671425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023741982848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766657.671430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766657.671432:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766657.671433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766657.671437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982976:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766657.671455:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766657.671461:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766657/real 1775766657] req@ffff90c218da0a80 x1862023741982976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766682 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766657.671468:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023741982976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766682 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766657.671474:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766657.671476:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766657.671478:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741982976:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775766658.695073:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766659.719127:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766660.743091:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766661.767073:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766662.790964:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766662.790972:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766662.790976:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766662.790985:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2551 00000100:00080000:3.0:1775766662.790995:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766662.791001:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766662.791042:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023741983104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775766662.791054:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766662.791072:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766662.791075:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766662.791077:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766662.791080:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2551 00000100:00080000:3.0:1775766662.791084:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766662.791088:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766662.791098:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766662.791105:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741983104:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766662.791115:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023741983232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766662.791151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766662.791156:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766662.791158:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766662.791163:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2551 00000100:00080000:3.0:1775766662.791180:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766662.791196:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766662.791200:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766662.791201:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766662.791214:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023741983360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766662.791222:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766662.791224:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766662.791226:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766662.791228:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2551 00000100:00080000:3.0:1775766662.791231:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766662.791234:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766662.791237:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766662.791246:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766662/real 1775766662] req@ffff90c218da0380 x1862023741983104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775766662.791248:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741983488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766662.791292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023741983104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766662.791301:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766662.791305:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766662.791309:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741983104:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766662.791334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766662.791358:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766662.791371:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766662/real 1775766662] req@ffff90c218da0700 x1862023741983232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766662.791381:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023741983232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766662.791398:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766662.791401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766662.791403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766662.791407:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766662.791421:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766662.791427:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766662/real 1775766662] req@ffff90c218da3480 x1862023741983360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766662.791435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023741983360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766662.791444:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766662.791446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766662.791448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766662.791451:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983488:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766662.791471:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766662.791477:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766662/real 1775766662] req@ffff90c218da0e00 x1862023741983488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766687 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766662.791484:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741983488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766687 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766662.791490:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766662.791492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766662.791494:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983488:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766663.815037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766664.839205:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766665.863060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766666.887070:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766667.910982:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766667.910991:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766667.910996:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766667.911007:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2556 00000100:00080000:2.0:1775766667.911020:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766667.911027:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766667.911060:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023741983616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766667.911107:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766667.911111:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766667.911112:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766667.911116:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2556 00000100:00080000:2.0:1775766667.911119:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766667.911123:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766667.911132:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766667.911148:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741983616:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766667.911148:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023741983744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766667.911165:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766667.911169:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766667.911172:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766667.911177:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2556 00000100:00080000:2.0:1775766667.911181:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766667.911186:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766667.911207:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766667.911217:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023741983872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766667.911225:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766667.911229:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766667.911232:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775766667.911233:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775766667.911240:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2556 00000100:00080000:2.0:1775766667.911245:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766667.911249:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766667.911254:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766667.911264:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023741984000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766667.911275:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766667/real 1775766667] req@ffff90c2034bd180 x1862023741983616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766667.911298:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023741983616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766667.911311:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766667.911317:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766667.911322:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741983616:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775766667.911348:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766667.911352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983744:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766667.911392:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766667.911402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766667/real 1775766667] req@ffff90c2034bdf80 x1862023741983744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766667.911410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023741983744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766667.911416:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766667.911418:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766667.911420:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766667.911424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983872:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766667.911435:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766667.911440:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766667/real 1775766667] req@ffff90c2034bf100 x1862023741983872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766667.911448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023741983872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766667.911452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766667.911454:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766667.911456:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741983872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766667.911459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984000:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766667.911468:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766667.911473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766667/real 1775766667] req@ffff90c2034bc700 x1862023741984000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766667.911479:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023741984000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766692 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766667.911485:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766667.911487:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766667.911489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984000:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766668.935150:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766669.959110:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766670.983125:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766672.007086:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766673.031049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766673.031063:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775766673.031080:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:0.0:1775766673.031087:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775766673.031094:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2561 00000100:00080000:3.0:1775766673.031109:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766673.031118:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775766673.031175:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023741984128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766673.031205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766673.031209:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775766673.031212:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766673.031217:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2561 00000100:00080000:3.0:1775766673.031221:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766673.031227:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766673.031251:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766673.031258:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741984128:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775766673.031269:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023741984256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766673.031280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766673.031283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775766673.031284:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766673.031288:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2561 00000100:00080000:3.0:1775766673.031291:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766673.031295:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766673.031299:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775766673.031310:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023741984384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775766673.031316:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775766673.031318:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775766673.031320:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775766673.031323:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2561 00000100:00080000:3.0:1775766673.031327:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775766673.031329:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775766673.031332:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775766673.031343:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775766673.031344:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023741984512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766673.031392:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766673/real 1775766673] req@ffff90c218da1880 x1862023741984128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766673.031415:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023741984128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766673.031427:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766673.031431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766673.031437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741984128:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766673.031463:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766673.031489:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766673.031503:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766673/real 1775766673] req@ffff90c218da3800 x1862023741984256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766673.031517:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023741984256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766673.031526:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766673.031529:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766673.031533:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766673.031540:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766673.031560:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766673.031571:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766673/real 1775766673] req@ffff90c218da2d80 x1862023741984384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766673.031601:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023741984384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766673.031610:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766673.031613:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766673.031617:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766673.031623:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984512:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766673.031644:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766673.031655:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766673/real 1775766673] req@ffff90c218da0e00 x1862023741984512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766673.031679:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023741984512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766673.031691:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766673.031694:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766673.031698:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984512:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766674.055169:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766675.079170:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766676.103114:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766677.127088:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766678.151053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766678.151063:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766678.151068:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766678.151080:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2566 00000100:00080000:1.0:1775766678.151092:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766678.151099:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775766678.151127:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766678.151154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741984640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766678.151209:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766678.151213:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766678.151215:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766678.151220:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2566 00000100:00080000:1.0:1775766678.151225:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766678.151231:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766678.151242:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766678.151258:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741984768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766678.151262:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741984640:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766678.151274:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766678.151278:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766678.151279:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766678.151283:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2566 00000100:00080000:1.0:1775766678.151286:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766678.151290:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766678.151294:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766678.151310:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741984896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766678.151322:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766678.151324:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766678.151326:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766678.151328:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2566 00000100:00080000:1.0:1775766678.151330:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766678.151332:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766678.151335:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766678.151343:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741985024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766678.151358:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766678.151418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766678/real 1775766678] req@ffff90c203496d80 x1862023741984640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766678.151442:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741984640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766678.151454:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766678.151459:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766678.151464:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741984640:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766678.151494:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766678.151537:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766678.151552:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766678/real 1775766678] req@ffff90c203497b80 x1862023741984768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766678.151569:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741984768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766678.151594:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766678.151597:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766678.151601:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766678.151608:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984896:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766678.151628:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766678.151639:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766678/real 1775766678] req@ffff90c203496300 x1862023741984896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766678.151652:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741984896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766678.151661:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766678.151664:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766678.151668:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741984896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766678.151675:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985024:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766678.151703:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766678.151714:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766678/real 1775766678] req@ffff90c203496a00 x1862023741985024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766678.151727:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741985024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766678.151739:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766678.151743:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766678.151747:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985024:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766679.175210:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766680.199070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766681.223123:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766682.247075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766683.271001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766683.271013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766683.271020:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766683.271032:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2571 00000100:00080000:0.0:1775766683.271045:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766683.271047:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766683.271053:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766683.271107:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741985152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766683.271134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766683.271137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766683.271138:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766683.271141:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2571 00000100:00080000:0.0:1775766683.271143:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766683.271146:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766683.271157:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766683.271171:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741985280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766683.271172:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741985152:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766683.271187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766683.271189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766683.271207:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766683.271209:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2571 00000100:00080000:0.0:1775766683.271211:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766683.271214:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766683.271218:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766683.271227:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741985408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766683.271240:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766683.271243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766683.271244:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766683.271247:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2571 00000100:00080000:0.0:1775766683.271250:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766683.271252:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766683.271255:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766683.271262:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741985536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766683.271283:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766683.271333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766683/real 1775766683] req@ffff90c203638700 x1862023741985152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766683.271357:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741985152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766683.271366:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766683.271380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766683.271384:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741985152:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766683.271409:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766683.271427:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766683.271436:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766683/real 1775766683] req@ffff90c203639500 x1862023741985280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766683.271445:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741985280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766683.271452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766683.271454:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766683.271456:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766683.271460:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766683.271471:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766683.271477:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766683/real 1775766683] req@ffff90c203639180 x1862023741985408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766683.271485:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741985408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766683.271496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766683.271498:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766683.271500:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766683.271503:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985536:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766683.271514:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766683.271519:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766683/real 1775766683] req@ffff90c203638000 x1862023741985536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766683.271526:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741985536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766683.271533:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766683.271535:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766683.271538:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985536:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775766684.295150:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766685.319059:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766686.343046:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766687.367071:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766688.391033:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766688.391044:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766688.391050:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766688.391061:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2576 00000100:00080000:0.0:1775766688.391073:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766688.391083:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766688.391099:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766688.391131:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741985664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766688.391153:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766688.391156:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766688.391157:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766688.391162:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2576 00000100:00080000:0.0:1775766688.391164:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766688.391168:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766688.391178:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766688.391189:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741985792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766688.391197:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766688.391203:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766688.391204:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766688.391206:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741985664:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766688.391211:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2576 00000100:00080000:0.0:1775766688.391222:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766688.391226:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766688.391229:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766688.391239:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741985920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766688.391246:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766688.391248:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766688.391250:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766688.391254:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2576 00000100:00080000:0.0:1775766688.391257:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766688.391260:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766688.391264:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766688.391273:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741986048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766688.391289:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766688.391328:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766688/real 1775766688] req@ffff90c20363aa00 x1862023741985664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766688.391349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741985664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766688.391361:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766688.391367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766688.391371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741985664:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766688.391396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766688.391415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766688.391424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766688/real 1775766688] req@ffff90c20363b100 x1862023741985792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766688.391433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741985792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766688.391439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766688.391441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766688.391443:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766688.391448:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766688.391459:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766688.391464:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766688/real 1775766688] req@ffff90c20363bb80 x1862023741985920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766688.391471:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741985920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766688.391476:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766688.391479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766688.391482:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741985920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766688.391499:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986048:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766688.391528:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766688.391539:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766688/real 1775766688] req@ffff90c20363a680 x1862023741986048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766688.391557:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741986048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766688.391567:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766688.391570:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766688.391573:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986048:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766689.415062:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775766690.439041:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766691.463002:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766692.487038:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766693.510959:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766693.510970:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766693.510975:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766693.510986:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2581 00000100:00080000:0.0:1775766693.511000:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766693.511008:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766693.511044:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766693.511052:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741986176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766693.511077:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766693.511082:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766693.511085:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766693.511089:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2581 00000100:00080000:0.0:1775766693.511094:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766693.511098:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766693.511120:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766693.511132:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741986176:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766693.511134:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741986304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766693.511146:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766693.511149:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766693.511151:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766693.511155:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2581 00000100:00080000:0.0:1775766693.511158:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766693.511163:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766693.511184:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766693.511195:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741986432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766693.511205:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766693.511208:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766693.511210:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766693.511214:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2581 00000100:00080000:0.0:1775766693.511218:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766693.511221:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766693.511227:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766693.511237:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741986560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766693.511267:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766693.511305:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766693.511333:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986560:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766694.535107:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766695.559080:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766696.583027:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766696.583209:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775766696.583266:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766696.583287:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766696.583335:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766696.583352:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766696.583370:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766696.583374:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766693/real 1775766696] req@ffff90c203638e00 x1862023741986304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766696.583403:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741986304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766696.583412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766696.583416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766696.583419:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766696.583425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766693/real 1775766696] req@ffff90c203639f80 x1862023741986432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766696.583433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741986432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766696.583439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766696.583442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766696.583445:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766696.583449:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766693/real 1775766696] req@ffff90c20363b800 x1862023741986560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766696.583460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741986560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766696.583471:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766696.583476:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766696.583479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986560:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766696.583486:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766693/real 1775766696] req@ffff90c203638380 x1862023741986176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766696.583499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741986176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766696.583507:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766696.583511:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766696.583514:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741986176:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766697.607063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766698.631001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766698.631012:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766698.631018:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766698.631031:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2587 00000100:00080000:0.0:1775766698.631045:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766698.631055:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766698.631108:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741986688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766698.631132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766698.631142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766698.631146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766698.631149:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766698.631154:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2587 00000100:00080000:0.0:1775766698.631160:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766698.631167:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766698.631180:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766698.631198:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741986816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766698.631203:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741986688:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766698.631233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766698.631237:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766698.631244:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766698.631257:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2587 00000100:00080000:0.0:1775766698.631266:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766698.631275:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766698.631285:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766698.631305:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741986944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766698.631318:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766698.631322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766698.631324:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775766698.631332:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766698.631334:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2587 00000100:00080000:0.0:1775766698.631346:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766698.631353:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766698.631363:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766698.631399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741987072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766698.631408:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766698/real 1775766698] req@ffff90c20363b480 x1862023741986688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766698.631454:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741986688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766698.631468:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766698.631474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766698.631478:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741986688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766698.631509:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766698.631535:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766698.631548:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766698/real 1775766698] req@ffff90c203638380 x1862023741986816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766698.631560:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741986816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766698.631569:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766698.631573:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766698.631576:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766698.631582:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766698.631607:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766698.631623:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766698/real 1775766698] req@ffff90c20363b800 x1862023741986944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766698.631641:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741986944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766698.631663:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766698.631668:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766698.631674:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741986944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766698.631685:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987072:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766698.631717:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766698.631734:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766698/real 1775766698] req@ffff90c203639f80 x1862023741987072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766723 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766698.631750:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741987072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766723 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766698.631785:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766698.631791:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766698.631796:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987072:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766699.655077:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766700.679042:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766701.703040:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766702.727009:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766703.750990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766703.751001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766703.751007:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775766703.751008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766703.751020:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2592 00000100:00080000:1.0:1775766703.751031:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766703.751037:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766703.751069:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741987200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766703.751096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766703.751099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766703.751101:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766703.751103:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2592 00000100:00080000:1.0:1775766703.751106:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766703.751109:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766703.751117:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766703.751127:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741987328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766703.751131:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741987200:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766703.751143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766703.751146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766703.751147:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766703.751151:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2592 00000100:00080000:1.0:1775766703.751153:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766703.751155:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766703.751159:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766703.751167:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741987456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766703.751173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766703.751176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766703.751178:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766703.751180:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2592 00000100:00080000:1.0:1775766703.751183:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766703.751187:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766703.751189:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766703.751194:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766703.751203:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741987584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766703.751223:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766703/real 1775766703] req@ffff90c203496680 x1862023741987200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766703.751238:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741987200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766703.751246:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766703.751249:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766703.751251:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741987200:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766703.751270:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766703.751290:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766703.751303:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766703/real 1775766703] req@ffff90c203494a80 x1862023741987328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766703.751317:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741987328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766703.751326:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766703.751330:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766703.751333:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766703.751338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766703.751353:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766703.751360:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766703/real 1775766703] req@ffff90c203495180 x1862023741987456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766703.751367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741987456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766703.751373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766703.751375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766703.751377:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766703.751380:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987584:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766703.751390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766703.751395:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766703/real 1775766703] req@ffff90c203495500 x1862023741987584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766728 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766703.751403:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741987584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766728 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766703.751409:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766703.751411:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766703.751413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987584:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766704.775023:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766705.799008:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766706.822996:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766707.847026:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766708.871033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766708.871038:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766708.871044:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766708.871049:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766708.871058:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2597 00000100:00080000:0.0:1775766708.871070:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766708.871076:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766708.871117:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741987712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766708.871150:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766708.871155:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766708.871157:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766708.871163:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2597 00000100:00080000:0.0:1775766708.871168:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766708.871174:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766708.871184:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766708.871186:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741987712:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766708.871200:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741987840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766708.871211:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766708.871215:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766708.871218:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766708.871222:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2597 00000100:00080000:0.0:1775766708.871237:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766708.871243:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766708.871249:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766708.871264:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766708.871269:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741987968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766708.871282:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766708.871287:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766708.871289:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766708.871293:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2597 00000100:00080000:0.0:1775766708.871298:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766708.871302:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766708.871309:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766708.871311:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766708/real 1775766708] req@ffff90c203638e00 x1862023741987712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766708.871330:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741988096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766708.871333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741987712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766708.871364:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766708.871370:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766708.871374:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741987712:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766708.871396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987840:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766708.871412:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766708.871421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766708/real 1775766708] req@ffff90c20363a680 x1862023741987840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766708.871430:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741987840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766708.871447:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766708.871451:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766708.871455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766708.871461:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766708.871484:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766708.871496:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766708/real 1775766708] req@ffff90c20363bb80 x1862023741987968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766708.871521:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741987968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766708.871527:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766708.871530:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766708.871532:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741987968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766708.871537:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988096:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766708.871554:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766708.871564:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766708/real 1775766708] req@ffff90c20363b100 x1862023741988096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766708.871577:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741988096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766733 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766708.871588:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766708.871592:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766708.871597:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988096:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766709.895044:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766710.919101:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766711.943107:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766712.967023:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766713.990968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766713.990979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766713.990984:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766713.990995:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2602 00000100:00080000:1.0:1775766713.991007:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766713.991017:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766713.991041:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766713.991055:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741988224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766713.991078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766713.991081:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766713.991082:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766713.991085:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2602 00000100:00080000:1.0:1775766713.991089:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766713.991092:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766713.991101:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766713.991113:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741988224:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766713.991114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741988352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766713.991127:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766713.991130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766713.991131:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766713.991134:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2602 00000100:00080000:1.0:1775766713.991137:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766713.991139:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766713.991142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766713.991150:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741988480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766713.991157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766713.991159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766713.991160:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766713.991163:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2602 00000100:00080000:1.0:1775766713.991167:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766713.991171:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766713.991177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766713.991182:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766713.991193:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741988608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766713.991225:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766713/real 1775766713] req@ffff90c203494700 x1862023741988224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766713.991240:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741988224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766713.991248:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766713.991251:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766713.991254:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741988224:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766713.991271:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766713.991286:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766713.991294:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766713/real 1775766713] req@ffff90c203495500 x1862023741988352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766713.991302:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741988352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766713.991308:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766713.991311:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766713.991313:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766713.991317:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766713.991326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766713.991333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766713/real 1775766713] req@ffff90c203495180 x1862023741988480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766713.991340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741988480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766713.991345:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766713.991346:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766713.991348:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766713.991352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988608:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766713.991361:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766713.991366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766713/real 1775766713] req@ffff90c203494a80 x1862023741988608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766713.991372:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741988608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766738 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766713.991379:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766713.991381:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766713.991383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988608:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766715.015025:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766716.039075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766717.063090:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766718.087139:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766719.110981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766719.110989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766719.110994:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766719.111002:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2607 00000100:00080000:1.0:1775766719.111013:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766719.111020:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766719.111068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741988736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766719.111079:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766719.111095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766719.111098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766719.111099:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766719.111103:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2607 00000100:00080000:1.0:1775766719.111105:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766719.111108:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766719.111117:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766719.111130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741988864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766719.111135:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741988736:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766719.111152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766719.111157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766719.111159:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766719.111164:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2607 00000100:00080000:1.0:1775766719.111169:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766719.111174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766719.111179:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766719.111190:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741988992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766719.111198:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775766719.111199:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766719.111203:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766719.111205:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766719.111209:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2607 00000100:00080000:1.0:1775766719.111214:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766719.111217:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766719.111221:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766719.111230:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741989120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766719.111237:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766719/real 1775766719] req@ffff90c203496680 x1862023741988736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766719.111252:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741988736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766719.111261:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766719.111264:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766719.111267:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741988736:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766719.111285:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988864:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766719.111299:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766719.111306:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766719/real 1775766719] req@ffff90c203496a00 x1862023741988864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766719.111314:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741988864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766719.111319:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766719.111321:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766719.111323:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766719.111327:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988992:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766719.111346:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766719.111351:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766719/real 1775766719] req@ffff90c203496300 x1862023741988992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766719.111358:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741988992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766719.111363:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766719.111364:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766719.111366:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741988992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766719.111370:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989120:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766719.111379:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766719.111384:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766719/real 1775766719] req@ffff90c203497b80 x1862023741989120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766719.111391:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741989120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766719.111397:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766719.111399:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766719.111401:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989120:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766720.135020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766721.159029:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766722.183066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766723.207096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766724.230951:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766724.230959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766724.230963:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766724.230972:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2612 00000100:00080000:1.0:1775766724.230983:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766724.230989:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766724.231016:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766724.231024:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741989248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766724.231045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766724.231048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766724.231050:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766724.231052:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2612 00000100:00080000:1.0:1775766724.231054:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766724.231057:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766724.231064:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766724.231073:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741989376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766724.231081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766724.231082:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741989248:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766724.231089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766724.231092:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766724.231095:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2612 00000100:00080000:1.0:1775766724.231097:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766724.231099:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766724.231102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766724.231111:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741989504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766724.231118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766724.231120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766724.231122:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766724.231125:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2612 00000100:00080000:1.0:1775766724.231127:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766724.231129:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766724.231131:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766724.231140:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741989632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766724.231147:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766724.231181:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766724/real 1775766724] req@ffff90c203496d80 x1862023741989248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766724.231195:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741989248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766724.231203:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766724.231205:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766724.231208:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741989248:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766724.231226:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766724.231239:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766724.231247:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766724/real 1775766724] req@ffff90c203495880 x1862023741989376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766724.231256:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741989376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766724.231261:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766724.231263:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766724.231265:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766724.231269:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766724.231277:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766724.231283:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766724/real 1775766724] req@ffff90c203497100 x1862023741989504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766724.231290:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741989504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766724.231295:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766724.231296:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766724.231298:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766724.231301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989632:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766724.231310:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766724.231315:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766724/real 1775766724] req@ffff90c203495c00 x1862023741989632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766724.231322:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741989632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766724.231332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766724.231334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766724.231336:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989632:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766725.255061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766726.279081:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766727.303068:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766728.327039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766729.351036:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766729.351045:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766729.351050:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766729.351058:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2617 00000100:00080000:0.0:1775766729.351069:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766729.351075:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766729.351122:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741989760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766729.351146:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766729.351151:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766729.351153:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766729.351156:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2617 00000100:00080000:0.0:1775766729.351160:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766729.351165:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766729.351182:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00100000:0.0:1775766729.351186:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766729.351196:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741989760:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766729.351199:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741989888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766729.351213:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766729.351215:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766729.351217:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766729.351219:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2617 00000100:00080000:0.0:1775766729.351221:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766729.351224:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766729.351227:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766729.351235:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741990016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766729.351241:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766729.351243:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766729.351245:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766729.351247:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2617 00000100:00080000:0.0:1775766729.351251:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766729.351253:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766729.351256:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766729.351261:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766729.351265:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741990144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766729.351301:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766729/real 1775766729] req@ffff90c20363aa00 x1862023741989760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766729.351321:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741989760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766729.351331:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766729.351335:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766729.351338:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741989760:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766729.351372:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766729.351397:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766729.351409:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766729/real 1775766729] req@ffff90c203638000 x1862023741989888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766729.351420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741989888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766729.351428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766729.351430:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766729.351434:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741989888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766729.351459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766729.351494:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766729.351504:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766729/real 1775766729] req@ffff90c203639180 x1862023741990016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766729.351516:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741990016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766729.351525:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766729.351529:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766729.351544:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766729.351550:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990144:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766729.351569:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766729.351577:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766729/real 1775766729] req@ffff90c203639500 x1862023741990144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766729.351585:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741990144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766729.351592:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766729.351594:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766729.351596:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990144:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766730.375081:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766731.399059:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766732.423025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766733.447094:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766734.470934:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766734.470942:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766734.470946:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766734.470955:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2622 00000100:00080000:0.0:1775766734.470966:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766734.470972:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766734.471011:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766734.471025:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741990272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766734.471082:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766734.471086:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766734.471090:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766734.471095:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2622 00000100:00080000:0.0:1775766734.471100:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766734.471104:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766734.471130:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766734.471143:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741990400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766734.471164:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766734.471170:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775766734.471171:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741990272:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766734.471172:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766734.471178:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2622 00000100:00080000:0.0:1775766734.471184:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766734.471188:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766734.471192:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766734.471202:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741990528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766734.471221:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766734.471224:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766734.471226:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766734.471229:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2622 00000100:00080000:0.0:1775766734.471232:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766734.471234:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766734.471237:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766734.471247:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741990656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766734.471257:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766734.471308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766734/real 1775766734] req@ffff90c203638700 x1862023741990272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766734.471326:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741990272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766734.471334:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766734.471338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766734.471342:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741990272:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766734.471362:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990400:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766734.471449:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766734.471463:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766734/real 1775766734] req@ffff90c203639500 x1862023741990400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766734.471473:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741990400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766734.471479:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766734.471481:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766734.471484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766734.471488:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990528:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766734.471504:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766734.471510:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766734/real 1775766734] req@ffff90c203639180 x1862023741990528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766734.471519:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741990528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766734.471529:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766734.471531:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766734.471534:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766734.471537:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990656:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766734.471548:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766734.471553:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766734/real 1775766734] req@ffff90c203638000 x1862023741990656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766734.471560:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741990656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766734.471567:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766734.471569:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766734.471571:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990656:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766735.495061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766736.519090:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766737.543029:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766738.567053:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766739.591017:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766739.591028:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766739.591035:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775766739.591042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766739.591047:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2628 00000100:00080000:1.0:1775766739.591059:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766739.591066:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766739.591129:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741990784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766739.591173:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766739.591178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766739.591181:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766739.591186:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2628 00000100:00080000:1.0:1775766739.591191:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766739.591195:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766739.591210:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766739.591228:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741990912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766739.591244:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766739.591253:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775766739.591256:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741990784:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766739.591257:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766739.591271:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2628 00000100:00080000:1.0:1775766739.591277:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766739.591281:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766739.591288:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766739.591329:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741991040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766739.591351:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766739.591354:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766739.591356:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766739.591359:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2628 00000100:00080000:1.0:1775766739.591363:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766739.591366:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766739.591381:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766739.591389:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741991168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766739.591400:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766739.591442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766739/real 1775766739] req@ffff90c203494380 x1862023741990784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766739.591460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741990784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766739.591473:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766739.591478:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766739.591483:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741990784:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766739.591514:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766739.591537:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766739.591546:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766739/real 1775766739] req@ffff90c203494e00 x1862023741990912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766739.591556:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741990912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766739.591562:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766739.591565:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766739.591567:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741990912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766739.591571:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766739.591582:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766739.591588:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766739/real 1775766739] req@ffff90c203495f80 x1862023741991040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766739.591596:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741991040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766739.591602:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766739.591603:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766739.591624:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766739.591631:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991168:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766739.591661:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766739.591680:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766739/real 1775766739] req@ffff90c203494000 x1862023741991168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766739.591704:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741991168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766739.591725:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766739.591732:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766739.591739:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991168:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766740.615178:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766741.639111:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766742.663080:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766743.687054:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766744.711000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766744.711029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766744.711034:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766744.711043:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2633 00010000:00010000:3.0:1775766744.711046:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766744.711054:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766744.711060:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766744.711099:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741991296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766744.711129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766744.711133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766744.711136:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766744.711140:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2633 00000100:00080000:0.0:1775766744.711144:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766744.711147:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766744.711156:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766744.711161:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741991296:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766744.711170:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741991424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766744.711178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766744.711182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766744.711183:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766744.711188:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2633 00000100:00080000:0.0:1775766744.711191:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766744.711195:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766744.711200:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766744.711212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741991552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766744.711221:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766744.711227:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766744.711231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766744.711233:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766744.711239:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2633 00000100:00080000:0.0:1775766744.711243:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766744.711248:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766744.711253:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766744.711260:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766744/real 1775766744] req@ffff90c20363aa00 x1862023741991296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766744.711272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741991680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766744.711300:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741991296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766744.711313:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766744.711317:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766744.711321:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741991296:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766744.711340:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991424:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766744.711357:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766744.711365:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766744/real 1775766744] req@ffff90c20363b100 x1862023741991424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766744.711373:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741991424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766744.711383:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766744.711385:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766744.711387:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766744.711391:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991552:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766744.711400:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766744.711405:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766744/real 1775766744] req@ffff90c20363bb80 x1862023741991552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766744.711412:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741991552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766744.711417:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766744.711419:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766744.711421:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766744.711424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991680:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766744.711440:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766744.711449:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766744/real 1775766744] req@ffff90c20363a680 x1862023741991680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766769 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766744.711461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741991680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766769 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766744.711472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766744.711476:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766744.711480:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991680:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766745.735088:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766746.759177:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766747.783027:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766748.807050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766749.830978:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766749.830993:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766749.831000:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766749.831013:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2638 00000100:00080000:0.0:1775766749.831026:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766749.831035:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766749.831040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766749.831082:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741991808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766749.831108:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766749.831111:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766749.831113:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766749.831116:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2638 00000100:00080000:0.0:1775766749.831120:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766749.831125:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766749.831137:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766749.831149:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741991808:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766749.831155:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741991936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766749.831172:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766749.831177:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766749.831179:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766749.831184:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2638 00000100:00080000:0.0:1775766749.831189:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766749.831193:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766749.831198:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766749.831209:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741992064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766749.831216:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775766749.831240:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766749.831242:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766749.831245:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766749.831250:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2638 00000100:00080000:0.0:1775766749.831256:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766749.831278:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766749.831285:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766749.831295:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766749/real 1775766749] req@ffff90c203638e00 x1862023741991808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766749.831304:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741992192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766749.831330:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741991808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766749.831340:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766749.831344:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766749.831348:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741991808:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766749.831371:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991936:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766749.831389:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766749.831397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766749/real 1775766749] req@ffff90c203639f80 x1862023741991936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766749.831416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741991936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766749.831422:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766749.831425:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766749.831427:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741991936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766749.831439:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766749.831450:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766749.831457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766749/real 1775766749] req@ffff90c20363b800 x1862023741992064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766749.831464:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741992064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766749.831469:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766749.831471:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766749.831473:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766749.831477:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992192:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766749.831487:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766749.831492:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766749/real 1775766749] req@ffff90c203638380 x1862023741992192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766749.831499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741992192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766774 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766749.831506:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766749.831507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766749.831509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992192:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766750.854999:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766751.879092:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766752.903088:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766753.927009:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766754.951005:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766754.951016:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766754.951021:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766754.951032:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2643 00010000:00010000:3.0:1775766754.951042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766754.951045:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766754.951053:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766754.951092:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023741992320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766754.951121:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766754.951125:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766754.951126:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766754.951130:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2643 00000100:00080000:1.0:1775766754.951135:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766754.951140:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766754.951150:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766754.951154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741992320:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766754.951169:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023741992448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766754.951210:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766754.951214:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766754.951216:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766754.951221:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2643 00000100:00080000:1.0:1775766754.951225:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766754.951229:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766754.951235:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775766754.951236:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766754.951252:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023741992576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766754.951264:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766754.951268:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766754.951271:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766754.951277:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2643 00000100:00100000:2.0:1775766754.951278:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766754/real 1775766754] req@ffff90c203497480 x1862023741992320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775766754.951282:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766754.951290:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766754.951299:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766754.951311:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023741992320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766754.951315:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023741992704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766754.951342:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766754.951347:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766754.951351:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741992320:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766754.951371:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766754.951390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766754.951398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766754/real 1775766754] req@ffff90c203494000 x1862023741992448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766754.951407:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023741992448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766754.951412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766754.951415:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766754.951417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766754.951420:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766754.951430:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766754.951436:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766754/real 1775766754] req@ffff90c203495f80 x1862023741992576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766754.951443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023741992576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766754.951447:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766754.951449:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766754.951451:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766754.951455:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992704:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766754.951465:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766754.951470:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766754/real 1775766754] req@ffff90c203494e00 x1862023741992704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766754.951477:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023741992704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766779 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766754.951484:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766754.951486:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766754.951487:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992704:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766755.975043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766756.999018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766758.023125:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766759.047157:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766760.070962:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766760.070970:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766760.070975:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766760.070983:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2648 00000100:00080000:0.0:1775766760.070994:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766760.071000:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766760.071026:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766760.071037:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741992832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766760.071060:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766760.071063:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766760.071065:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766760.071068:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2648 00000100:00080000:0.0:1775766760.071071:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766760.071074:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766760.071082:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766760.071093:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741992960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766760.071100:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766760.071103:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766760.071105:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766760.071105:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741992832:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766760.071113:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2648 00000100:00080000:0.0:1775766760.071117:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766760.071131:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766760.071137:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766760.071149:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741993088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766760.071159:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766760.071163:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766760.071164:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766760.071169:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2648 00000100:00080000:0.0:1775766760.071174:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766760.071178:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766760.071184:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766760.071198:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741993216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766760.071224:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766760.071249:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766760.071265:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993216:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766761.095010:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766762.119061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766763.143064:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766763.143197:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775766763.143241:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766763.143264:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766763.143314:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766763.143325:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766763.143340:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766763.143375:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766760/real 1775766763] req@ffff90c203638380 x1862023741992960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766763.143404:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741992960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766763.143413:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766763.143416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766763.143420:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741992960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766763.143426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766760/real 1775766763] req@ffff90c20363b800 x1862023741993088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766763.143434:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741993088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766763.143439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766763.143441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766763.143443:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766763.143448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766760/real 1775766763] req@ffff90c203639f80 x1862023741993216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766763.143455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741993216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766763.143465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766763.143467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766763.143469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993216:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766763.143473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766760/real 1775766763] req@ffff90c20363b480 x1862023741992832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766763.143482:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741992832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766763.143487:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766763.143492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766763.143495:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741992832:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766764.167087:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766765.190944:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766765.190953:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766765.190958:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766765.190966:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2653 00000100:00080000:0.0:1775766765.190978:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766765.190984:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766765.190999:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766765.191016:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741993344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766765.191038:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766765.191040:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766765.191042:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766765.191045:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2653 00000100:00080000:0.0:1775766765.191047:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766765.191050:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766765.191058:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766765.191082:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741993472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766765.191082:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741993344:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766765.191097:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766765.191100:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766765.191101:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766765.191105:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2653 00000100:00080000:0.0:1775766765.191108:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766765.191112:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766765.191117:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766765.191127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741993600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766765.191134:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766765.191137:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766765.191138:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766765.191141:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2653 00000100:00080000:0.0:1775766765.191143:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766765.191146:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766765.191149:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766765.191153:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766765.191161:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741993728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766765.191192:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766765/real 1775766765] req@ffff90c203638e00 x1862023741993344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766765.191210:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741993344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766765.191218:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766765.191221:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766765.191224:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741993344:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766765.191245:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766765.191268:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766765.191279:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766765/real 1775766765] req@ffff90c20363a680 x1862023741993472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766765.191292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741993472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766765.191300:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766765.191304:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766765.191307:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766765.191314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766765.191335:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766765.191346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766765/real 1775766765] req@ffff90c20363bb80 x1862023741993600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766765.191357:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741993600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766765.191362:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766765.191364:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766765.191366:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766765.191370:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993728:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766765.191391:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766765.191401:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766765/real 1775766765] req@ffff90c20363b100 x1862023741993728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766765.191414:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741993728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766765.191424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766765.191427:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766765.191430:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993728:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766766.215044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766767.239012:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766768.263123:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766769.287006:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766770.310991:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766770.311002:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766770.311007:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766770.311035:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2658 00000100:00080000:0.0:1775766770.311049:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766770.311056:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766770.311063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766770.311102:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741993856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766770.311126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766770.311130:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766770.311131:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766770.311135:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2658 00000100:00080000:0.0:1775766770.311137:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766770.311140:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766770.311150:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766770.311160:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741993984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766770.311171:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766770.311173:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741993856:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766770.311178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766770.311187:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766770.311194:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2658 00000100:00080000:0.0:1775766770.311198:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766770.311203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766770.311209:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766770.311226:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741994112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766770.311238:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766770.311242:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766770.311244:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766770.311248:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2658 00000100:00080000:0.0:1775766770.311253:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766770.311254:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766770.311264:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766770.311271:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766770.311285:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741994240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766770.311297:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766770/real 1775766770] req@ffff90c20363aa00 x1862023741993856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766770.311316:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741993856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766770.311330:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766770.311333:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766770.311336:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741993856:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766770.311355:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766770.311372:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766770.311380:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766770/real 1775766770] req@ffff90c203638000 x1862023741993984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766770.311388:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741993984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766770.311393:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766770.311395:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766770.311397:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741993984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766770.311402:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766770.311413:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766770.311418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766770/real 1775766770] req@ffff90c203639180 x1862023741994112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766770.311425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741994112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766770.311430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766770.311432:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766770.311434:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766770.311438:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994240:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766770.311449:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766770.311454:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766770/real 1775766770] req@ffff90c203639500 x1862023741994240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766770.311475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741994240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766770.311485:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766770.311489:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766770.311492:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994240:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766771.335055:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766772.359171:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766773.383095:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766774.407005:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766775.430977:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766775.430989:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766775.430993:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766775.431004:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2663 00000100:00080000:0.0:1775766775.431018:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766775.431026:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766775.431059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766775.431068:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023741994368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766775.431096:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766775.431099:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766775.431100:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766775.431103:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2663 00000100:00080000:0.0:1775766775.431106:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766775.431109:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766775.431118:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766775.431130:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741994496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766775.431142:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741994368:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766775.431150:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766775.431153:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766775.431155:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766775.431157:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2663 00000100:00080000:0.0:1775766775.431159:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766775.431168:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766775.431172:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766775.431181:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741994624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766775.431187:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766775.431190:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766775.431191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766775.431193:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2663 00000100:00080000:0.0:1775766775.431196:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766775.431198:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766775.431201:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766775.431207:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741994752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766775.431225:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766775.431267:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766775/real 1775766775] req@ffff90c203638700 x1862023741994368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766775.431284:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023741994368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766775.431291:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766775.431295:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766775.431297:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741994368:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766775.431316:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994496:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766775.431334:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766775.431342:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766775/real 1775766775] req@ffff90c203639500 x1862023741994496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766775.431350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741994496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766775.431355:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766775.431358:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766775.431360:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766775.431363:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766775.431373:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766775.431378:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766775/real 1775766775] req@ffff90c203639180 x1862023741994624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766775.431385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741994624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766775.431390:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766775.431392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766775.431393:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766775.431397:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994752:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766775.431416:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766775.431422:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766775/real 1775766775] req@ffff90c203638000 x1862023741994752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766775.431429:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741994752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766775.431439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766775.431441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766775.431443:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741994752:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766776.455085:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766777.479071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766778.503095:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766779.527117:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766780.551045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766780.551053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766780.551057:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766780.551065:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2669 00000100:00080000:1.0:1775766780.551076:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766780.551080:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766780.551083:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766780.551122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023741994880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766780.551152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766780.551156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766780.551158:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766780.551163:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2669 00000100:00080000:1.0:1775766780.551167:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766780.551171:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766780.551181:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766780.551194:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023741995008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766780.551198:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741994880:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766780.551211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766780.551213:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766780.551215:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766780.551218:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2669 00000100:00080000:1.0:1775766780.551222:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766780.551225:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766780.551228:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766780.551238:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023741995136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766780.551245:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766780.551247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766780.551248:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766780.551250:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2669 00000100:00080000:1.0:1775766780.551254:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766780.551256:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766780.551259:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766780.551264:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766780.551268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023741995264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766780.551302:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766780/real 1775766780] req@ffff90c203494380 x1862023741994880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766780.551350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023741994880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766780.551365:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766780.551372:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766780.551378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741994880:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766780.551409:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766780.551444:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766780.551464:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766780/real 1775766780] req@ffff90c203495c00 x1862023741995008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766780.551484:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023741995008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766780.551498:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766780.551503:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766780.551509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766780.551519:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766780.551563:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766780.551593:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766780/real 1775766780] req@ffff90c203497100 x1862023741995136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766780.551606:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023741995136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766780.551614:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766780.551617:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766780.551620:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766780.551626:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995264:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766780.551646:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766780.551655:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766780/real 1775766780] req@ffff90c203495880 x1862023741995264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766780.551665:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023741995264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766780.551674:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766780.551677:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766780.551679:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995264:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766781.575092:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766782.599076:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766783.623103:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766784.647060:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766785.671036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766785.671046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766785.671052:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766785.671065:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2674 00000100:00080000:1.0:1775766785.671077:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766785.671085:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766785.671097:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766785.671134:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023741995392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766785.671161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766785.671164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766785.671165:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766785.671168:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2674 00000100:00080000:1.0:1775766785.671171:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766785.671174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766785.671185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766785.671196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741995520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766785.671207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775766785.671210:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741995392:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766785.671211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766785.671218:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766785.671231:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2674 00000100:00080000:1.0:1775766785.671234:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766785.671236:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766785.671239:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766785.671254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741995648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766785.671261:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766785.671264:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766785.671266:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766785.671268:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2674 00000100:00080000:1.0:1775766785.671271:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766785.671276:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766785.671282:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766785.671297:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741995776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766785.671301:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766785.671348:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766785/real 1775766785] req@ffff90c203496d80 x1862023741995392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766785.671376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023741995392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766785.671384:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766785.671388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766785.671391:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741995392:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766785.671411:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995520:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766785.671425:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766785.671432:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766785/real 1775766785] req@ffff90c203497b80 x1862023741995520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766785.671440:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741995520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766785.671445:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766785.671447:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766785.671449:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766785.671453:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995648:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766785.671462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766785.671468:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766785/real 1775766785] req@ffff90c203496300 x1862023741995648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766785.671475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741995648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766785.671484:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766785.671486:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766785.671489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766785.671493:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995776:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766785.671503:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766785.671508:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766785/real 1775766785] req@ffff90c203496a00 x1862023741995776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766810 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766785.671514:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741995776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766810 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766785.671528:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766785.671530:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766785.671532:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741995776:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766786.695075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766787.719055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766788.743014:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766789.767100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766790.791022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766790.791033:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766790.791041:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766790.791053:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2679 00010000:00010000:3.0:1775766790.791060:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766790.791067:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766790.791077:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766790.791129:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741995904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766790.791162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766790.791166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766790.791168:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766790.791171:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2679 00000100:00080000:1.0:1775766790.791174:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766790.791178:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766790.791189:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766790.791205:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741996032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766790.791210:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741995904:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766790.791239:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766790.791243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766790.791245:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766790.791251:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2679 00000100:00080000:1.0:1775766790.791254:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766790.791257:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766790.791261:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766790.791273:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741996160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766790.791286:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766790.791290:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766790.791293:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766790.791296:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2679 00000100:00080000:1.0:1775766790.791299:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766790.791302:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766790.791306:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766790.791309:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766790.791322:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741996288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766790.791358:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766790/real 1775766790] req@ffff90c203496680 x1862023741995904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766790.791377:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741995904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766790.791387:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766790.791391:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766790.791394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741995904:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766790.791418:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766790.791436:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766790.791445:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766790/real 1775766790] req@ffff90c203494a80 x1862023741996032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766790.791454:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741996032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766790.791461:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766790.791464:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766790.791466:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766790.791471:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766790.791483:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766790.791490:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766790/real 1775766790] req@ffff90c203495180 x1862023741996160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766790.791498:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741996160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766790.791504:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766790.791507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766790.791509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766790.791513:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996288:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766790.791525:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766790.791531:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766790/real 1775766790] req@ffff90c203495500 x1862023741996288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766815 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766790.791551:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741996288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766815 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766790.791560:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766790.791562:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766790.791564:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996288:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766791.815085:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766792.839067:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766793.863136:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766794.887042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766795.911038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766795.911048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766795.911053:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766795.911063:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2684 00000100:00080000:1.0:1775766795.911077:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766795.911100:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766795.911142:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023741996416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766795.911142:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766795.911172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766795.911176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766795.911178:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766795.911183:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2684 00000100:00080000:1.0:1775766795.911187:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766795.911191:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766795.911203:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766795.911212:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741996416:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766795.911223:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023741996544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766795.911237:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766795.911241:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766795.911244:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766795.911248:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2684 00000100:00080000:1.0:1775766795.911253:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766795.911257:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766795.911262:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766795.911272:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766795.911276:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023741996672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766795.911284:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766795.911287:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766795.911289:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766795.911292:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2684 00000100:00080000:1.0:1775766795.911295:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766795.911297:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766795.911302:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766795.911311:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766795/real 1775766795] req@ffff90c203494700 x1862023741996416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766795.911312:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023741996800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766795.911376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023741996416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766795.911388:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766795.911392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766795.911396:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741996416:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766795.911420:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996544:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766795.911454:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766795.911465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766795/real 1775766795] req@ffff90c203495500 x1862023741996544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766795.911496:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023741996544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766795.911513:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766795.911517:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766795.911521:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766795.911527:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766795.911558:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766795.911564:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766795/real 1775766795] req@ffff90c203495180 x1862023741996672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766795.911572:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023741996672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766795.911577:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766795.911579:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766795.911581:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766795.911584:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996800:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766795.911595:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766795.911600:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766795/real 1775766795] req@ffff90c203494a80 x1862023741996800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766795.911607:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023741996800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766820 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766795.911614:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766795.911616:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766795.911618:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741996800:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766796.935058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766797.959105:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766798.983163:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766800.007128:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766801.031048:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766801.031060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766801.031068:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766801.031082:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2689 00000100:00080000:1.0:1775766801.031095:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766801.031104:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766801.031127:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766801.031159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023741996928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766801.031195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766801.031198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766801.031200:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766801.031204:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2689 00000100:00080000:1.0:1775766801.031207:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766801.031210:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766801.031221:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766801.031248:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023741997056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766801.031253:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741996928:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766801.031267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766801.031270:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766801.031272:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766801.031276:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2689 00000100:00080000:1.0:1775766801.031278:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766801.031281:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766801.031286:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766801.031296:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023741997184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766801.031311:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766801.031314:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766801.031316:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766801.031319:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2689 00000100:00080000:1.0:1775766801.031322:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766801.031324:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766801.031329:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766801.031338:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023741997312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766801.031346:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766801.031396:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766801/real 1775766801] req@ffff90c203496680 x1862023741996928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766801.031416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023741996928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766801.031427:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766801.031431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766801.031435:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741996928:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766801.031461:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997056:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766801.031480:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766801.031494:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766801/real 1775766801] req@ffff90c203496a00 x1862023741997056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766801.031510:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023741997056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766801.031521:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766801.031526:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766801.031530:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766801.031538:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997184:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766801.031564:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766801.031576:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766801/real 1775766801] req@ffff90c203496300 x1862023741997184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766801.031586:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023741997184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766801.031594:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766801.031596:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766801.031599:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766801.031604:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997312:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766801.031620:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766801.031626:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766801/real 1775766801] req@ffff90c203497b80 x1862023741997312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766801.031636:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023741997312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766801.031644:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766801.031646:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766801.031649:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997312:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766802.055136:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766803.078993:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766804.103117:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766805.127100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766806.150999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766806.151007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766806.151013:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766806.151033:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2694 00000100:00080000:0.0:1775766806.151043:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766806.151050:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766806.151111:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741997440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766806.151156:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766806.151160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766806.151164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766806.151166:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766806.151170:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2694 00000100:00080000:0.0:1775766806.151174:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766806.151178:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766806.151189:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766806.151199:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741997568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766806.151208:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741997440:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766806.151213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766806.151239:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766806.151240:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766806.151244:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2694 00000100:00080000:0.0:1775766806.151247:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766806.151250:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766806.151253:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766806.151268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741997696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766806.151274:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766806.151277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766806.151278:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766806.151282:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2694 00000100:00080000:0.0:1775766806.151311:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766806.151314:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766806.151324:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766806.151333:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741997824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766806.151384:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766806.151429:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766806/real 1775766806] req@ffff90c20363aa00 x1862023741997440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766806.151463:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741997440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766806.151477:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766806.151482:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766806.151487:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741997440:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766806.151515:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766806.151552:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766806.151561:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766806/real 1775766806] req@ffff90c20363b100 x1862023741997568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766806.151570:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741997568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766806.151576:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766806.151578:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766806.151580:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766806.151585:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766806.151596:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766806.151602:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766806/real 1775766806] req@ffff90c20363bb80 x1862023741997696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766806.151619:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741997696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766806.151624:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766806.151626:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766806.151628:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766806.151633:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997824:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766806.151672:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766806.151679:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766806/real 1775766806] req@ffff90c20363a680 x1862023741997824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766806.151687:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741997824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766806.151695:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766806.151697:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766806.151699:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741997824:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766807.175182:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766808.199052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766809.223045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766810.247026:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766811.271072:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766811.271085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766811.271092:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766811.271106:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2699 00000100:00080000:0.0:1775766811.271123:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775766811.271123:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766811.271145:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766811.271200:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741997952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766811.271248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766811.271253:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766811.271255:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766811.271261:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2699 00000100:00080000:0.0:1775766811.271265:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766811.271270:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766811.271282:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766811.271300:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741998080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766811.271316:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741997952:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766811.271320:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766811.271333:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766811.271338:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766811.271344:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2699 00000100:00080000:0.0:1775766811.271349:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766811.271354:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766811.271361:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766811.271380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741998208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766811.271399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766811.271406:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766811.271409:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766811.271417:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2699 00000100:00080000:0.0:1775766811.271424:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766811.271434:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766811.271437:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766811.271462:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766811.271490:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741998336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766811.271506:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766811/real 1775766811] req@ffff90c203638e00 x1862023741997952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766811.271538:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741997952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766811.271555:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766811.271562:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766811.271568:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741997952:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766811.271647:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998080:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766811.271680:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766811.271703:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766811/real 1775766811] req@ffff90c20363b480 x1862023741998080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766811.271721:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741998080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766811.271732:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766811.271736:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766811.271740:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766811.271747:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998208:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766811.271772:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766811.271782:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766811/real 1775766811] req@ffff90c203639f80 x1862023741998208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766811.271796:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741998208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766811.271805:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766811.271808:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766811.271812:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766811.271819:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998336:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766811.271836:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766811.271846:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766811/real 1775766811] req@ffff90c20363b800 x1862023741998336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766811.271858:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741998336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766811.271910:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766811.271915:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766811.271919:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998336:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766812.295057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766813.319024:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766814.343137:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766815.367012:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766816.390938:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766816.390946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766816.390950:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766816.390958:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2704 00000100:00080000:0.0:1775766816.390970:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766816.390976:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766816.391010:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023741998464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766816.391013:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766816.391036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766816.391040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766816.391042:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766816.391045:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2704 00000100:00080000:0.0:1775766816.391048:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766816.391051:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766816.391059:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766816.391068:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741998464:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766816.391072:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023741998592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766816.391081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766816.391084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766816.391085:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766816.391087:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2704 00000100:00080000:0.0:1775766816.391089:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766816.391091:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766816.391094:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766816.391101:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023741998720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766816.391107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766816.391109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766816.391110:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766816.391113:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2704 00000100:00080000:0.0:1775766816.391116:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766816.391118:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766816.391121:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766816.391130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023741998848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766816.391135:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766816.391171:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766816/real 1775766816] req@ffff90c203638380 x1862023741998464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766816.391196:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023741998464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766816.391203:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766816.391206:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766816.391209:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741998464:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766816.391227:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998592:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766816.391249:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766816.391257:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766816/real 1775766816] req@ffff90c20363b800 x1862023741998592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766816.391265:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023741998592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766816.391270:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766816.391272:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766816.391274:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766816.391279:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766816.391289:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766816.391294:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766816/real 1775766816] req@ffff90c203639f80 x1862023741998720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766816.391301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023741998720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766816.391306:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766816.391307:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766816.391309:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766816.391313:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998848:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766816.391322:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766816.391327:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766816/real 1775766816] req@ffff90c20363b480 x1862023741998848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766816.391333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023741998848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766816.391339:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766816.391341:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766816.391343:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741998848:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766817.415113:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766818.439055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766819.463030:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766820.487044:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766821.510978:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766821.510988:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766821.510993:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766821.511004:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2709 00000100:00080000:0.0:1775766821.511016:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766821.511024:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766821.511026:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766821.511067:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023741998976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766821.511107:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766821.511111:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766821.511113:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766821.511119:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2709 00000100:00080000:0.0:1775766821.511124:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766821.511128:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766821.511137:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741998976:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775766821.511139:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766821.511159:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023741999104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766821.511173:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766821.511177:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766821.511180:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766821.511185:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2709 00000100:00080000:0.0:1775766821.511189:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766821.511195:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766821.511201:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766821.511207:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766821.511216:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023741999232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766821.511228:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766821.511233:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766821.511235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766821.511240:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2709 00000100:00080000:0.0:1775766821.511245:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766821.511248:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766821/real 1775766821] req@ffff90c203638e00 x1862023741998976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766821.511253:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766821.511264:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766821.511278:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023741999360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766821.511283:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023741998976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766821.511305:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766821.511309:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766821.511314:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741998976:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766821.511333:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766821.511350:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766821.511359:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766821/real 1775766821] req@ffff90c20363a680 x1862023741999104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766821.511367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023741999104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766821.511373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766821.511375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766821.511377:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766821.511381:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766821.511398:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766821.511403:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766821/real 1775766821] req@ffff90c20363bb80 x1862023741999232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766821.511417:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023741999232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766821.511427:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766821.511429:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766821.511432:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766821.511435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999360:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766821.511447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766821.511452:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766821/real 1775766821] req@ffff90c20363b100 x1862023741999360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766821.511459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023741999360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766821.511466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766821.511468:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766821.511469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999360:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766822.535171:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766823.559025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766824.583107:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766825.607111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766826.631005:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766826.631016:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766826.631022:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766826.631033:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2715 00000100:00080000:0.0:1775766826.631047:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766826.631055:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766826.631096:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023741999488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766826.631130:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775766826.631135:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766826.631136:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766826.631149:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766826.631155:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2715 00000100:00080000:0.0:1775766826.631161:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766826.631169:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766826.631170:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741999488:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775766826.631182:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766826.631201:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023741999616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766826.631216:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766826.631220:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766826.631223:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766826.631228:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2715 00000100:00080000:0.0:1775766826.631232:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766826.631236:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766826.631242:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766826.631254:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023741999744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766826.631266:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766826.631270:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766826.631272:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766826.631277:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2715 00000100:00080000:0.0:1775766826.631281:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766826.631286:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766826.631291:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766826.631299:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766826.631306:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023741999872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766826.631345:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766826.631366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999872:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766827.655059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766828.679034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766829.703162:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775766829.703269:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775766829.703315:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775766829.703332:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766829.703370:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766829.703382:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766829.703410:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766829.703476:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766826/real 1775766829] req@ffff90c203639500 x1862023741999872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766829.703500:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023741999872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766829.703512:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766829.703515:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766829.703519:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999872:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775766829.703525:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766826/real 1775766829] req@ffff90c203638000 x1862023741999616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766829.703535:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023741999616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766829.703543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766829.703547:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766829.703551:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766829.703557:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766826/real 1775766829] req@ffff90c203639180 x1862023741999744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766829.703569:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023741999744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766829.703576:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766829.703580:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766829.703584:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023741999744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766829.703591:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766826/real 1775766829] req@ffff90c20363aa00 x1862023741999488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766851 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766829.703602:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023741999488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766851 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766829.703609:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766829.703612:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766829.703616:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023741999488:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766830.727050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766831.751025:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766831.751028:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766831.751041:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766831.751049:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766831.751065:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2720 00000100:00080000:0.0:1775766831.751082:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766831.751092:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766831.751166:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742000000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766831.751215:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766831.751220:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766831.751223:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766831.751231:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2720 00000100:00080000:0.0:1775766831.751239:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766831.751247:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766831.751265:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766831.751297:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742000128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766831.751330:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742000000:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766831.751335:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766831.751341:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766831.751344:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766831.751350:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2720 00000100:00080000:0.0:1775766831.751355:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766831.751361:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766831.751386:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766831.751412:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742000256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766831.751425:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766831.751430:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766831.751433:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766831.751440:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2720 00000100:00080000:0.0:1775766831.751446:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766831.751452:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766831.751460:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766831.751462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766831.751483:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742000384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766831.751532:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766831/real 1775766831] req@ffff90c203638700 x1862023742000000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766831.751560:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742000000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766831.751591:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766831.751596:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766831.751601:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742000000:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766831.751636:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766831.751668:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766831.751684:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766831/real 1775766831] req@ffff90c20363aa00 x1862023742000128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766831.751701:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742000128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766831.751712:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766831.751716:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766831.751720:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766831.751729:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766831.751757:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766831.751771:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766831/real 1775766831] req@ffff90c203639180 x1862023742000256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766831.751788:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742000256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766831.751800:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766831.751811:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766831.751815:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766831.751824:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000384:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766831.751851:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766831.751865:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766831/real 1775766831] req@ffff90c203638000 x1862023742000384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766856 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766831.751918:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742000384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766856 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766831.751933:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766831.751937:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766831.751941:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000384:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766832.775116:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766833.799158:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766834.823045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766835.846995:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766836.871072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766836.871103:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766836.871113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766836.871117:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766836.871125:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2725 00000100:00080000:1.0:1775766836.871136:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766836.871141:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766836.871176:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742000512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766836.871202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766836.871205:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766836.871206:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766836.871209:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2725 00000100:00080000:1.0:1775766836.871211:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766836.871214:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766836.871223:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766836.871235:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742000640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766836.871243:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742000512:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766836.871252:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766836.871256:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766836.871258:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766836.871264:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2725 00000100:00080000:1.0:1775766836.871268:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766836.871274:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766836.871280:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766836.871297:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742000768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766836.871308:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766836.871311:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766836.871313:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766836.871332:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2725 00000100:00080000:1.0:1775766836.871337:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775766836.871338:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766836.871349:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766836.871357:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766836.871372:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742000896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766836.871400:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766836/real 1775766836] req@ffff90c203496d80 x1862023742000512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766836.871426:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742000512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766836.871437:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766836.871441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766836.871444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742000512:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766836.871470:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766836.871489:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766836.871499:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766836/real 1775766836] req@ffff90c203495880 x1862023742000640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766836.871507:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742000640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766836.871514:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766836.871516:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766836.871518:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766836.871522:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766836.871532:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766836.871538:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766836/real 1775766836] req@ffff90c203497100 x1862023742000768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766836.871545:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742000768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766836.871555:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766836.871557:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766836.871559:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766836.871562:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000896:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766836.871572:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766836.871577:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766836/real 1775766836] req@ffff90c203495c00 x1862023742000896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766836.871584:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742000896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766861 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766836.871600:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766836.871601:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766836.871603:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742000896:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766837.895042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766838.918998:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766839.943027:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766840.966994:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766841.991003:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766841.991012:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766841.991016:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766841.991024:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2730 00000100:00080000:0.0:1775766841.991037:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766841.991043:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766841.991063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766841.991090:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742001024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766841.991124:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766841.991127:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766841.991129:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766841.991132:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2730 00000100:00080000:0.0:1775766841.991135:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766841.991138:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766841.991145:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766841.991152:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742001024:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766841.991156:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742001152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766841.991166:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766841.991168:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766841.991169:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766841.991172:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2730 00000100:00080000:0.0:1775766841.991174:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766841.991176:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766841.991179:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766841.991187:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742001280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766841.991194:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766841.991196:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766841.991208:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766841.991210:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2730 00000100:00080000:0.0:1775766841.991213:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766841.991215:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766841.991219:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766841.991225:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742001408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766841.991229:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766841.991292:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766841/real 1775766841] req@ffff90c203639500 x1862023742001024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766841.991308:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742001024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766841.991315:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766841.991318:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766841.991322:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742001024:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766841.991353:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001152:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766841.991369:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766841.991376:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766841/real 1775766841] req@ffff90c20363b100 x1862023742001152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766841.991384:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742001152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766841.991389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766841.991391:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766841.991393:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766841.991397:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766841.991406:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766841.991411:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766841/real 1775766841] req@ffff90c20363bb80 x1862023742001280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766841.991418:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742001280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766841.991423:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766841.991424:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766841.991426:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766841.991429:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001408:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766841.991439:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766841.991443:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766841/real 1775766841] req@ffff90c20363a680 x1862023742001408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766841.991451:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742001408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766866 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766841.991457:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766841.991458:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766841.991460:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001408:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766843.015082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766844.039124:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766845.063009:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766846.087061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766847.110988:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766847.111012:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766847.111018:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766847.111023:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766847.111031:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2735 00000100:00080000:1.0:1775766847.111042:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766847.111048:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766847.111082:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742001536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766847.111105:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766847.111108:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766847.111109:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766847.111112:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2735 00000100:00080000:1.0:1775766847.111114:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766847.111117:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766847.111124:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766847.111134:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742001664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766847.111139:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742001536:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766847.111145:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766847.111149:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766847.111151:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766847.111156:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2735 00000100:00080000:1.0:1775766847.111160:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766847.111164:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766847.111170:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766847.111183:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742001792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766847.111200:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766847.111201:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766847.111206:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766847.111222:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766847.111228:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2735 00000100:00080000:1.0:1775766847.111233:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766847.111236:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766847.111240:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766847.111254:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766847/real 1775766847] req@ffff90c203494380 x1862023742001536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775766847.111257:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742001920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766847.111285:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742001536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766847.111294:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766847.111298:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766847.111302:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742001536:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766847.111321:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766847.111342:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766847.111354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766847/real 1775766847] req@ffff90c203494e00 x1862023742001664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766847.111368:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742001664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766847.111384:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766847.111388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766847.111392:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766847.111396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766847.111411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766847.111419:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766847/real 1775766847] req@ffff90c203495f80 x1862023742001792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766847.111426:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742001792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766847.111431:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766847.111434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766847.111436:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766847.111439:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001920:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766847.111449:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766847.111454:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766847/real 1775766847] req@ffff90c203494000 x1862023742001920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766847.111461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742001920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766847.111467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766847.111469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766847.111487:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742001920:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766848.135034:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766849.159068:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766850.183138:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766851.206983:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766852.231072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766852.231086:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766852.231100:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766852.231106:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766852.231120:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2740 00000100:00080000:0.0:1775766852.231137:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766852.231148:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766852.231255:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742002048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766852.231310:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766852.231314:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766852.231316:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766852.231320:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2740 00000100:00080000:0.0:1775766852.231324:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766852.231327:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766852.231338:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766852.231349:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742002176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766852.231360:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766852.231363:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766852.231365:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766852.231372:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742002048:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766852.231373:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2740 00000100:00080000:0.0:1775766852.231376:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766852.231383:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766852.231392:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766852.231402:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742002304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766852.231410:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766852.231413:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766852.231414:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766852.231417:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2740 00000100:00080000:0.0:1775766852.231420:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766852.231423:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766852.231426:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766852.231437:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742002432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766852.231465:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766852.231512:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766852/real 1775766852] req@ffff90c203638e00 x1862023742002048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766852.231536:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742002048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766852.231549:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766852.231554:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766852.231559:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742002048:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766852.231588:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002176:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766852.231626:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766852.231640:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766852/real 1775766852] req@ffff90c20363b480 x1862023742002176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766852.231655:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742002176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766852.231665:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766852.231669:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766852.231674:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766852.231681:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002304:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766852.231704:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766852.231717:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766852/real 1775766852] req@ffff90c203639f80 x1862023742002304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766852.231729:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742002304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766852.231739:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766852.231743:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766852.231747:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766852.231754:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002432:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766852.231786:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766852.231797:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766852/real 1775766852] req@ffff90c20363b800 x1862023742002432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766852.231811:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742002432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766852.231824:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766852.231828:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766852.231832:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002432:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766853.255004:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766854.279008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766855.303142:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766856.327108:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766857.351022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766857.351036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766857.351043:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766857.351069:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2745 00000100:00080000:0.0:1775766857.351084:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766857.351094:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766857.351173:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742002560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766857.351177:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766857.351199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766857.351202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766857.351204:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766857.351207:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2745 00000100:00080000:0.0:1775766857.351210:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766857.351212:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766857.351222:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766857.351234:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742002688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766857.351243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766857.351246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775766857.351247:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742002560:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766857.351248:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766857.351253:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2745 00000100:00080000:0.0:1775766857.351258:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766857.351262:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766857.351265:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766857.351272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742002816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766857.351279:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766857.351283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766857.351284:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766857.351287:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2745 00000100:00080000:0.0:1775766857.351290:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766857.351293:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766857.351296:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766857.351304:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742002944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766857.351334:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766857.351384:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766857/real 1775766857] req@ffff90c203638380 x1862023742002560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766857.351407:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742002560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766857.351420:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766857.351425:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766857.351430:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742002560:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766857.351459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766857.351484:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766857.351498:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766857/real 1775766857] req@ffff90c20363b800 x1862023742002688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766857.351512:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742002688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766857.351522:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766857.351526:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766857.351529:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766857.351536:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766857.351559:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766857.351570:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766857/real 1775766857] req@ffff90c203639f80 x1862023742002816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766857.351583:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742002816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766857.351592:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766857.351596:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766857.351600:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766857.351607:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002944:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766857.351629:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766857.351639:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766857/real 1775766857] req@ffff90c20363b480 x1862023742002944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766857.351652:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742002944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766857.351664:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766857.351667:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766857.351671:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742002944:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766858.375154:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766859.399138:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766860.423158:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766861.446994:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766862.471130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775766862.471139:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766862.471142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766862.471148:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766862.471168:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2750 00000100:00080000:1.0:1775766862.471180:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766862.471186:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766862.471221:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742003072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766862.471249:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766862.471263:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766862.471265:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766862.471267:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2750 00000100:00080000:1.0:1775766862.471269:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766862.471271:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766862.471280:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766862.471291:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742003200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766862.471299:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766862.471302:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766862.471308:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766862.471309:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742003072:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766862.471315:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2750 00000100:00080000:1.0:1775766862.471338:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766862.471342:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766862.471345:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766862.471357:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742003328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766862.471364:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766862.471368:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766862.471371:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766862.471376:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2750 00000100:00080000:1.0:1775766862.471382:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766862.471407:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766862.471417:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766862.471427:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766862.471452:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742003456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766862.471483:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766862/real 1775766862] req@ffff90c203497480 x1862023742003072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766862.471503:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742003072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766862.471518:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766862.471523:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766862.471527:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742003072:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766862.471580:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766862.471603:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766862.471625:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766862/real 1775766862] req@ffff90c203494000 x1862023742003200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766862.471637:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742003200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766862.471644:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766862.471647:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766862.471650:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766862.471655:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766862.471670:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766862.471688:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766862/real 1775766862] req@ffff90c203495f80 x1862023742003328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766862.471697:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742003328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766862.471720:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766862.471722:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766862.471724:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766862.471728:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003456:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766862.471740:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766862.471745:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766862/real 1775766862] req@ffff90c203494e00 x1862023742003456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766862.471753:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742003456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766862.471780:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766862.471784:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766862.471788:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003456:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766863.495110:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766864.519045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766865.543041:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766866.566996:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766867.591008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766867.591018:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766867.591025:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766867.591036:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2756 00000100:00080000:1.0:1775766867.591048:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766867.591055:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766867.591078:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766867.591093:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742003584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766867.591116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766867.591119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766867.591120:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766867.591123:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2756 00000100:00080000:1.0:1775766867.591125:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766867.591127:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766867.591135:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766867.591147:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742003712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766867.591151:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742003584:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766867.591169:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766867.591172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766867.591174:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766867.591179:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2756 00000100:00080000:1.0:1775766867.591184:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766867.591188:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766867.591194:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766867.591206:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742003840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766867.591218:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766867.591222:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1775766867.591223:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766867.591225:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766867.591230:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2756 00000100:00080000:1.0:1775766867.591233:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766867.591236:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766867.591240:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766867.591249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742003968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766867.591261:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766867/real 1775766867] req@ffff90c203494380 x1862023742003584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766867.591289:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742003584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766867.591297:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766867.591301:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766867.591305:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742003584:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766867.591339:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003712:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766867.591364:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766867.591376:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766867/real 1775766867] req@ffff90c203495c00 x1862023742003712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766867.591385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742003712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766867.591398:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766867.591400:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766867.591402:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766867.591407:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003840:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766867.591421:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766867.591428:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766867/real 1775766867] req@ffff90c203497100 x1862023742003840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766867.591435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742003840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766867.591446:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766867.591448:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766867.591450:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766867.591454:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003968:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766867.591466:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766867.591471:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766867/real 1775766867] req@ffff90c203495880 x1862023742003968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766867.591478:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742003968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766867.591485:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766867.591487:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766867.591489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742003968:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766868.615036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766869.639048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766870.663066:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766871.687082:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766872.711045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766872.711053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766872.711058:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766872.711066:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2761 00000100:00080000:1.0:1775766872.711077:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766872.711083:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766872.711118:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742004096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766872.711125:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766872.711138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766872.711143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766872.711146:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766872.711151:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2761 00000100:00080000:1.0:1775766872.711154:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766872.711157:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766872.711166:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766872.711176:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742004096:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766872.711177:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742004224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766872.711199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766872.711202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766872.711203:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766872.711207:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2761 00000100:00080000:1.0:1775766872.711209:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766872.711211:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766872.711215:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766872.711223:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742004352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766872.711230:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766872.711234:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766872.711235:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766872.711238:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2761 00000100:00080000:1.0:1775766872.711240:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766872.711242:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766872.711246:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766872.711252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742004480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766872.711273:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766872.711315:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766872/real 1775766872] req@ffff90c203496d80 x1862023742004096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766872.711331:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742004096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766872.711342:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766872.711347:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766872.711351:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742004096:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766872.711382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766872.711408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766872.711419:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766872/real 1775766872] req@ffff90c203497b80 x1862023742004224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766872.711428:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742004224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766872.711434:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766872.711437:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766872.711439:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766872.711444:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766872.711456:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766872.711462:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766872/real 1775766872] req@ffff90c203496300 x1862023742004352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766872.711470:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742004352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766872.711479:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766872.711482:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766872.711484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766872.711489:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004480:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766872.711500:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766872.711509:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766872/real 1775766872] req@ffff90c203496a00 x1862023742004480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766897 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766872.711521:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742004480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766897 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766872.711532:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766872.711536:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766872.711539:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004480:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766873.735038:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766874.759132:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766875.783101:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766876.807098:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766877.830988:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766877.830995:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766877.831000:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766877.831010:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2766 00000100:00080000:0.0:1775766877.831021:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766877.831028:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766877.831065:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742004608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766877.831081:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766877.831089:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766877.831093:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766877.831096:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766877.831100:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2766 00000100:00080000:0.0:1775766877.831104:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766877.831110:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766877.831120:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766877.831139:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742004736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766877.831144:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742004608:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766877.831164:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766877.831167:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766877.831170:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766877.831174:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2766 00000100:00080000:0.0:1775766877.831179:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766877.831182:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766877.831187:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766877.831198:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742004864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766877.831205:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766877.831207:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766877.831209:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766877.831211:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2766 00000100:00080000:0.0:1775766877.831214:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766877.831219:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766877.831225:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766877.831231:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766877.831236:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742004992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766877.831280:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766877/real 1775766877] req@ffff90c203638e00 x1862023742004608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766877.831297:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742004608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766877.831306:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766877.831310:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766877.831313:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742004608:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766877.831337:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004736:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766877.831352:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766877.831360:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766877/real 1775766877] req@ffff90c20363a680 x1862023742004736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766877.831369:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742004736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766877.831374:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766877.831377:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766877.831379:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766877.831383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004864:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766877.831393:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766877.831398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766877/real 1775766877] req@ffff90c20363bb80 x1862023742004864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766877.831405:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742004864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766877.831410:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766877.831412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766877.831413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766877.831417:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004992:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766877.831427:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766877.831432:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766877/real 1775766877] req@ffff90c20363b100 x1862023742004992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766877.831441:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742004992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766902 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766877.831448:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766877.831450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766877.831451:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742004992:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766878.855050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766879.879037:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766880.903015:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766881.927011:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766882.950972:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766882.950985:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766882.950989:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766882.950997:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2771 00000100:00080000:0.0:1775766882.951009:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766882.951015:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766882.951041:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766882.951053:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742005120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766882.951082:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766882.951086:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766882.951088:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766882.951093:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2771 00000100:00080000:0.0:1775766882.951097:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766882.951102:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766882.951112:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766882.951113:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742005120:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766882.951128:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742005248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766882.951137:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766882.951140:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766882.951141:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766882.951145:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2771 00000100:00080000:0.0:1775766882.951148:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766882.951150:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766882.951154:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766882.951160:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742005376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766882.951167:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766882.951170:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766882.951171:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766882.951174:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2771 00000800:00000100:2.0:1775766882.951178:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766882.951179:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766882.951184:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766882.951187:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766882.951196:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742005504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766882.951216:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766882/real 1775766882] req@ffff90c203639500 x1862023742005120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766882.951230:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742005120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766882.951239:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766882.951241:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766882.951244:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742005120:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766882.951261:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766882.951275:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766882.951283:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766882/real 1775766882] req@ffff90c203638000 x1862023742005248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766882.951290:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742005248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766882.951300:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766882.951303:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766882.951305:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766882.951308:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766882.951317:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766882.951323:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766882/real 1775766882] req@ffff90c203639180 x1862023742005376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766882.951329:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742005376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766882.951334:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766882.951336:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766882.951338:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766882.951341:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005504:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766882.951350:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766882.951355:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766882/real 1775766882] req@ffff90c20363aa00 x1862023742005504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766882.951361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742005504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766907 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766882.951368:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766882.951370:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766882.951372:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005504:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766883.975026:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766884.999046:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766886.022984:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766887.047030:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766888.071044:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766888.071054:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766888.071060:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766888.071071:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2776 00000100:00080000:1.0:1775766888.071083:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766888.071091:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766888.071114:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766888.071146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742005632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766888.071169:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766888.071172:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766888.071173:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766888.071191:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2776 00000100:00080000:1.0:1775766888.071195:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766888.071198:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766888.071209:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766888.071222:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742005760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766888.071223:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742005632:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766888.071239:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766888.071242:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766888.071243:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766888.071246:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2776 00000100:00080000:1.0:1775766888.071248:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766888.071251:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766888.071255:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766888.071265:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742005888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766888.071277:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766888.071280:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766888.071282:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766888.071285:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2776 00000100:00080000:1.0:1775766888.071288:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766888.071290:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766888.071294:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766888.071301:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766888.071305:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742006016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766888.071347:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766888/real 1775766888] req@ffff90c203496680 x1862023742005632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766888.071364:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742005632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766888.071373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766888.071377:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766888.071380:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742005632:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766888.071406:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766888.071432:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766888.071476:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766888/real 1775766888] req@ffff90c203494a80 x1862023742005760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766888.071490:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742005760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766888.071497:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766888.071500:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766888.071518:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766888.071524:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766888.071545:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766888.071557:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766888/real 1775766888] req@ffff90c203495180 x1862023742005888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766888.071584:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742005888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766888.071594:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766888.071600:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766888.071605:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742005888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766888.071615:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006016:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766888.071640:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766888.071653:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766888/real 1775766888] req@ffff90c203495500 x1862023742006016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766888.071670:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742006016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766888.071680:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766888.071683:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766888.071702:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006016:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766889.095071:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766890.119055:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766891.143071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766892.167125:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766893.191031:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766893.191065:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766893.191080:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766893.191087:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766893.191104:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2781 00000100:00080000:1.0:1775766893.191122:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766893.191133:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766893.191197:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742006144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766893.191243:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766893.191249:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766893.191253:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766893.191261:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2781 00000100:00080000:1.0:1775766893.191269:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766893.191283:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766893.191284:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742006144:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775766893.191309:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766893.191337:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742006272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766893.191358:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766893.191364:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766893.191367:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766893.191396:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2781 00000100:00080000:1.0:1775766893.191404:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766893.191406:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006272:192.168.202.155@tcp:38:kworker.0 00000100:00080000:1.0:1775766893.191419:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766893.191430:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766893.191457:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742006400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766893.191488:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766893.191494:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766893.191498:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766893.191506:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2781 00000100:00080000:1.0:1775766893.191514:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775766893.191519:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006400:192.168.202.155@tcp:38:kworker.0 00000100:00080000:1.0:1775766893.191527:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766893.191540:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766893.191565:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742006528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766893.191624:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006528:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766894.215069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766895.239000:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766896.263251:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775766896.263311:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775766896.263372:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775766896.263398:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766896.263449:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766896.263485:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775766896.263508:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775766896.263525:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766893/real 1775766896] req@ffff90c203495180 x1862023742006400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766896.263582:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742006400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766896.263595:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766896.263600:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766896.263604:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766896.263611:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766893/real 1775766896] req@ffff90c203495500 x1862023742006272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766896.263623:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742006272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766896.263631:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766896.263634:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766896.263637:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766896.263643:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766893/real 1775766896] req@ffff90c203494700 x1862023742006144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766896.263653:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742006144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766896.263660:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766896.263663:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766896.263666:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742006144:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766896.263691:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766893/real 1775766896] req@ffff90c203494a80 x1862023742006528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766896.263738:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742006528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766896.263753:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766896.263758:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766896.263763:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006528:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766897.287072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766898.311000:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766898.311012:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766898.311018:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766898.311029:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2786 00000100:00080000:1.0:1775766898.311056:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766898.311062:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766898.311070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766898.311102:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742006656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766898.311126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766898.311130:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766898.311132:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766898.311136:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2786 00000100:00080000:1.0:1775766898.311140:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766898.311145:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766898.311168:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766898.311177:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742006656:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775766898.311185:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742006784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766898.311195:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766898.311198:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766898.311199:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766898.311204:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2786 00000100:00080000:1.0:1775766898.311208:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766898.311212:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766898.311218:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766898.311230:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742006912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766898.311247:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775766898.311252:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775766898.311254:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766898.311257:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766898.311262:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2786 00000100:00080000:1.0:1775766898.311267:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766898.311271:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766898.311277:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766898.311291:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742007040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766898.311298:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766898/real 1775766898] req@ffff90c203496680 x1862023742006656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766898.311321:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742006656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766898.311332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766898.311336:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766898.311340:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742006656:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766898.311376:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766898.311401:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766898.311414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766898/real 1775766898] req@ffff90c203496a00 x1862023742006784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766898.311425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742006784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766898.311433:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766898.311436:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766898.311450:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766898.311456:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766898.311476:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766898.311486:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766898/real 1775766898] req@ffff90c203496300 x1862023742006912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766898.311497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742006912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766898.311504:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766898.311507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766898.311510:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742006912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766898.311516:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007040:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766898.311537:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766898.311548:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766898/real 1775766898] req@ffff90c203497b80 x1862023742007040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766898.311558:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742007040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766898.311569:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766898.311572:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766898.311576:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007040:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766899.335075:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766900.359132:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766901.382998:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766902.407027:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766903.430964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766903.430973:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766903.430977:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766903.430987:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2791 00000100:00080000:1.0:1775766903.430998:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766903.431004:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766903.431042:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742007168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766903.431042:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766903.431068:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766903.431072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766903.431073:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766903.431076:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2791 00000100:00080000:1.0:1775766903.431079:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766903.431082:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766903.431108:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766903.431119:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742007296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766903.431123:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742007168:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766903.431139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766903.431142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766903.431143:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766903.431147:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2791 00000100:00080000:1.0:1775766903.431150:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766903.431153:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766903.431156:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766903.431167:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742007424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766903.431174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766903.431177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766903.431179:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766903.431182:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2791 00000100:00080000:1.0:1775766903.431187:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766903.431194:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775766903.431195:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775766903.431204:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766903.431235:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742007552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766903.431258:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766903/real 1775766903] req@ffff90c203496d80 x1862023742007168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766903.431278:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742007168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766903.431304:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766903.431308:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766903.431312:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742007168:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766903.431331:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766903.431351:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766903.431361:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766903/real 1775766903] req@ffff90c203495880 x1862023742007296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766903.431371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742007296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766903.431377:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766903.431379:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766903.431382:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766903.431386:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007424:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766903.431398:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766903.431419:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766903/real 1775766903] req@ffff90c203497100 x1862023742007424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766903.431435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742007424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766903.431444:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766903.431465:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766903.431469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766903.431477:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007552:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766903.431500:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766903.431509:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766903/real 1775766903] req@ffff90c203495c00 x1862023742007552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766903.431518:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742007552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766903.431534:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766903.431537:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766903.431539:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007552:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766904.455040:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766905.478989:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766906.503040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766907.527040:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766908.550962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766908.550971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766908.550975:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766908.550984:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2797 00000100:00080000:1.0:1775766908.550996:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766908.551002:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766908.551018:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766908.551039:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742007680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766908.551062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766908.551065:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766908.551066:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766908.551069:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2797 00000100:00080000:1.0:1775766908.551071:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766908.551074:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766908.551082:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766908.551091:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742007808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766908.551095:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742007680:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766908.551103:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766908.551106:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766908.551107:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766908.551110:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2797 00000100:00080000:1.0:1775766908.551112:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766908.551114:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766908.551118:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766908.551125:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742007936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766908.551136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766908.551138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766908.551140:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766908.551142:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2797 00000100:00080000:1.0:1775766908.551145:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766908.551147:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766908.551150:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766908.551158:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775766908.551159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742008064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766908.551194:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766908/real 1775766908] req@ffff90c203494380 x1862023742007680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766908.551210:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742007680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766908.551218:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766908.551221:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766908.551224:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742007680:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766908.551250:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007808:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766908.551266:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766908.551274:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766908/real 1775766908] req@ffff90c203494e00 x1862023742007808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766908.551282:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742007808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766908.551288:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766908.551290:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766908.551292:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766908.551296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007936:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766908.551306:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766908.551311:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766908/real 1775766908] req@ffff90c203495f80 x1862023742007936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766908.551318:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742007936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766908.551323:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766908.551325:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766908.551327:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742007936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766908.551331:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008064:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766908.551341:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766908.551346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766908/real 1775766908] req@ffff90c203494000 x1862023742008064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766908.551353:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742008064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766908.551360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766908.551362:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766908.551364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008064:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766909.575048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766910.599032:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766911.623211:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766912.647102:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766913.671015:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766913.671027:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766913.671033:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766913.671045:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2802 00000100:00080000:0.0:1775766913.671059:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766913.671067:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766913.671094:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766913.671104:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742008192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766913.671129:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766913.671132:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766913.671134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766913.671138:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2802 00000100:00080000:0.0:1775766913.671141:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766913.671145:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766913.671154:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766913.671164:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742008192:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766913.671167:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742008320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766913.671179:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766913.671181:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766913.671182:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766913.671185:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2802 00000100:00080000:0.0:1775766913.671188:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766913.671190:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766913.671193:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766913.671202:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742008448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766913.671209:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766913.671211:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766913.671213:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766913.671215:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2802 00000100:00080000:0.0:1775766913.671218:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766913.671221:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766913.671224:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775766913.671250:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766913.671254:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742008576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766913.671300:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766913/real 1775766913] req@ffff90c203638700 x1862023742008192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766913.671319:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742008192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766913.671346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766913.671349:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766913.671352:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742008192:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766913.671371:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766913.671387:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766913.671395:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766913/real 1775766913] req@ffff90c20363aa00 x1862023742008320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766913.671404:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742008320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766913.671410:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766913.671421:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766913.671424:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766913.671428:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766913.671440:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766913.671447:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766913/real 1775766913] req@ffff90c203639180 x1862023742008448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766913.671455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742008448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766913.671461:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766913.671463:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766913.671465:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766913.671468:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008576:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766913.671479:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766913.671484:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766913/real 1775766913] req@ffff90c203638000 x1862023742008576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766938 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766913.671492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742008576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766938 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766913.671504:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766913.671506:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766913.671508:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008576:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766914.695026:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766915.719178:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766916.743030:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766917.767036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766918.790958:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766918.790967:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766918.790972:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766918.790980:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2807 00000100:00080000:0.0:1775766918.790991:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766918.790997:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766918.790998:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766918.791033:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742008704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766918.791058:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766918.791061:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766918.791063:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766918.791065:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2807 00000100:00080000:0.0:1775766918.791068:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766918.791072:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766918.791082:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766918.791096:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742008704:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766918.791097:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742008832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766918.791113:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766918.791117:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766918.791119:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766918.791124:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2807 00000100:00080000:0.0:1775766918.791128:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766918.791133:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766918.791139:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766918.791152:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742008960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766918.791160:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766918.791163:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766918.791166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766918.791167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766918.791172:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2807 00000100:00080000:0.0:1775766918.791176:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766918.791179:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766918.791183:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766918.791194:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742009088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766918.791199:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766918/real 1775766918] req@ffff90c203639500 x1862023742008704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766918.791217:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742008704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766918.791225:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766918.791228:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766918.791231:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742008704:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766918.791256:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008832:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766918.791271:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766918.791278:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766918/real 1775766918] req@ffff90c20363b100 x1862023742008832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766918.791286:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742008832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766918.791292:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766918.791294:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766918.791296:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766918.791301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008960:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766918.791311:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766918.791317:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766918/real 1775766918] req@ffff90c20363bb80 x1862023742008960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766918.791324:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742008960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766918.791329:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766918.791330:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766918.791332:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742008960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766918.791335:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009088:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766918.791345:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766918.791350:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766918/real 1775766918] req@ffff90c20363a680 x1862023742009088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766943 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766918.791357:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742009088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766943 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766918.791363:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766918.791365:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766918.791366:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009088:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766919.815054:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766920.839091:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766921.863077:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766922.887041:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766923.911086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775766923.911093:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766923.911095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766923.911105:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766923.911115:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2812 00000100:00080000:1.0:1775766923.911126:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766923.911132:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775766923.911168:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742009216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766923.911194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766923.911197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766923.911198:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766923.911201:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2812 00000100:00080000:1.0:1775766923.911203:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766923.911206:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766923.911215:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766923.911227:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742009344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766923.911235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742009216:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766923.911240:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766923.911248:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766923.911249:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766923.911253:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2812 00000100:00080000:1.0:1775766923.911254:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766923.911256:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766923.911260:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766923.911272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742009472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766923.911278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766923.911281:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766923.911282:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766923.911285:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2812 00000100:00080000:1.0:1775766923.911288:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766923.911290:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766923.911296:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766923.911302:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742009600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766923.911321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766923.911362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766923/real 1775766923] req@ffff90c203497480 x1862023742009216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766923.911379:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742009216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766923.911388:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766923.911392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766923.911395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742009216:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766923.911436:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766923.911461:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766923.911475:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766923/real 1775766923] req@ffff90c203494000 x1862023742009344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766923.911489:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742009344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766923.911495:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766923.911498:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766923.911500:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766923.911504:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766923.911520:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766923.911528:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766923/real 1775766923] req@ffff90c203495f80 x1862023742009472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766923.911535:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742009472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766923.911545:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766923.911548:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766923.911550:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766923.911554:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009600:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766923.911564:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766923.911569:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766923/real 1775766923] req@ffff90c203494e00 x1862023742009600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766923.911585:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742009600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766948 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766923.911593:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766923.911595:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766923.911597:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009600:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766924.935084:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766925.959144:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766926.983045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766928.007033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766929.031041:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766929.031056:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766929.031062:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766929.031073:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2817 00000100:00080000:0.0:1775766929.031085:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766929.031091:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766929.031118:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766929.031136:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742009728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766929.031161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766929.031164:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766929.031167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766929.031170:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2817 00000100:00080000:0.0:1775766929.031173:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766929.031176:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766929.031187:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766929.031198:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742009728:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766929.031199:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742009856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766929.031212:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766929.031214:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766929.031215:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766929.031218:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2817 00000100:00080000:0.0:1775766929.031220:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766929.031222:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766929.031225:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766929.031250:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742009984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766929.031257:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766929.031260:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766929.031262:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766929.031265:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2817 00000100:00080000:0.0:1775766929.031269:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766929.031272:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766929.031277:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766929.031287:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742010112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766929.031315:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766929.031369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766929/real 1775766929] req@ffff90c203638e00 x1862023742009728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766929.031386:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742009728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766929.031396:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766929.031400:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766929.031403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742009728:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766929.031426:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766929.031442:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766929.031450:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766929/real 1775766929] req@ffff90c20363b480 x1862023742009856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766929.031458:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742009856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766929.031464:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766929.031466:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766929.031468:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766929.031472:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766929.031482:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766929.031487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766929/real 1775766929] req@ffff90c203639f80 x1862023742009984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766929.031494:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742009984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766929.031500:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766929.031501:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766929.031503:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742009984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766929.031506:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010112:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766929.031516:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766929.031521:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766929/real 1775766929] req@ffff90c20363b800 x1862023742010112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766929.031528:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742010112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766929.031536:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766929.031537:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766929.031539:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010112:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766930.055074:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766931.079081:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766932.103114:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766933.127021:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766934.150952:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766934.150971:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766934.151007:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766934.151020:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2822 00000100:00080000:0.0:1775766934.151045:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766934.151054:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766934.151060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766934.151096:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742010240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766934.151123:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766934.151128:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766934.151130:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766934.151134:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2822 00000100:00080000:0.0:1775766934.151138:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766934.151142:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766934.151150:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766934.151154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742010240:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766934.151166:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742010368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766934.151175:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766934.151178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766934.151180:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766934.151183:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2822 00000100:00080000:0.0:1775766934.151186:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766934.151189:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766934.151192:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766934.151199:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742010496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766934.151206:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766934.151209:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766934.151210:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775766934.151214:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766934.151217:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2822 00000100:00080000:0.0:1775766934.151221:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766934.151223:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766934.151227:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766934.151235:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742010624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766934.151254:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766934/real 1775766934] req@ffff90c203638380 x1862023742010240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766934.151271:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742010240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766934.151278:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766934.151281:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766934.151284:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742010240:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766934.151319:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010368:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766934.151341:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766934.151354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766934/real 1775766934] req@ffff90c20363b800 x1862023742010368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766934.151364:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742010368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766934.151374:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766934.151376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766934.151378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766934.151382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010496:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766934.151396:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766934.151403:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766934/real 1775766934] req@ffff90c203639f80 x1862023742010496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766934.151410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742010496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766934.151415:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766934.151416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766934.151418:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766934.151422:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010624:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766934.151431:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766934.151436:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766934/real 1775766934] req@ffff90c20363b480 x1862023742010624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766934.151442:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742010624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766934.151448:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766934.151450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766934.151453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010624:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766935.175078:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766936.199120:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766937.223050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766938.247084:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766939.271013:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766939.271023:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766939.271029:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766939.271040:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2827 00000100:00080000:2.0:1775766939.271077:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766939.271099:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766939.271142:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be680 x1862023742010752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766939.271174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766939.271178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766939.271181:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766939.271186:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2827 00000100:00080000:2.0:1775766939.271190:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766939.271195:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766939.271205:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766939.271222:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bed80 x1862023742010880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766939.271224:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742010752:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775766939.271241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766939.271264:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766939.271266:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766939.271270:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2827 00000100:00080000:2.0:1775766939.271274:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766939.271278:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766939.271284:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766939.271298:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023742011008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766939.271307:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766939.271310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766939.271311:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766939.271315:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2827 00000100:00080000:2.0:1775766939.271320:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766939.271324:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766939.271329:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775766939.271332:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766939.271342:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023742011136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766939.271375:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766939/real 1775766939] req@ffff90c2034be680 x1862023742010752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766939.271395:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be680 x1862023742010752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766939.271406:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766939.271411:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766939.271416:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742010752:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775766939.271442:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766939.271446:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766939.271473:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766939.271486:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766939/real 1775766939] req@ffff90c2034bed80 x1862023742010880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766939.271499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bed80 x1862023742010880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766939.271507:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766939.271511:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766939.271514:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742010880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766939.271520:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766939.271540:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766939.271551:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766939/real 1775766939] req@ffff90c2034bc700 x1862023742011008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766939.271563:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023742011008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766939.271581:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766939.271584:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766939.271588:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766939.271594:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011136:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766939.271615:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766939.271627:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766939/real 1775766939] req@ffff90c2034bf100 x1862023742011136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766939.271639:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023742011136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766939.271648:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766939.271652:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766939.271655:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011136:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766940.295017:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766941.319062:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766942.343154:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766943.367192:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766944.391044:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766944.391055:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766944.391060:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766944.391073:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2832 00010000:00010000:3.0:1775766944.391081:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766944.391089:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766944.391098:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766944.391152:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742011264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766944.391180:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766944.391183:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766944.391185:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766944.391189:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2832 00000100:00080000:0.0:1775766944.391203:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766944.391207:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766944.391218:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766944.391230:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742011392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766944.391237:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742011264:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766944.391253:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766944.391255:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766944.391257:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766944.391261:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2832 00000100:00080000:0.0:1775766944.391263:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766944.391265:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766944.391269:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766944.391278:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742011520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766944.391285:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766944.391288:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766944.391289:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766944.391291:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2832 00000100:00080000:0.0:1775766944.391294:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766944.391296:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766944.391299:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766944.391306:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742011648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766944.391325:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766944.391367:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766944/real 1775766944] req@ffff90c203638e00 x1862023742011264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766944.391383:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742011264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766944.391392:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766944.391397:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766944.391400:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742011264:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766944.391426:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011392:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766944.391440:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766944.391448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766944/real 1775766944] req@ffff90c20363a680 x1862023742011392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766944.391457:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742011392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766944.391462:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766944.391464:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766944.391466:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766944.391470:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011520:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766944.391480:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766944.391485:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766944/real 1775766944] req@ffff90c20363bb80 x1862023742011520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766944.391493:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742011520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766944.391498:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766944.391499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766944.391501:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766944.391504:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011648:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766944.391514:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766944.391519:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766944/real 1775766944] req@ffff90c20363b100 x1862023742011648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766944.391527:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742011648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766944.391534:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766944.391536:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766944.391537:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011648:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766945.416095:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766946.439222:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766947.463075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766948.487142:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766949.511003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766949.511014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766949.511019:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766949.511030:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2837 00000100:00080000:0.0:1775766949.511044:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766949.511052:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766949.511090:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766949.511110:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742011776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766949.511135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766949.511150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766949.511153:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766949.511158:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2837 00000100:00080000:0.0:1775766949.511163:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766949.511167:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766949.511191:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742011776:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775766949.511192:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766949.511213:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742011904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766949.511223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766949.511226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766949.511228:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766949.511231:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2837 00000100:00080000:0.0:1775766949.511235:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766949.511238:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766949.511243:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766949.511252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742012032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766949.511258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766949.511261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766949.511263:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775766949.511268:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766949.511271:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2837 00000100:00080000:0.0:1775766949.511276:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766949.511279:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766949.511283:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766949.511300:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742012160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766949.511313:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766949/real 1775766949] req@ffff90c203639500 x1862023742011776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766949.511330:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742011776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766949.511346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766949.511350:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766949.511353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742011776:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766949.511375:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011904:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766949.511391:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766949.511399:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766949/real 1775766949] req@ffff90c203638000 x1862023742011904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766949.511416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742011904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766949.511428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766949.511431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766949.511433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742011904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766949.511436:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766949.511447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766949.511452:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766949/real 1775766949] req@ffff90c203639180 x1862023742012032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766949.511459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742012032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766949.511465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766949.511466:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766949.511468:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766949.511472:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012160:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766949.511490:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766949.511495:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766949/real 1775766949] req@ffff90c20363aa00 x1862023742012160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766949.511501:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742012160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766949.511509:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766949.511511:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766949.511513:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012160:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766950.535099:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766951.559049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766952.583179:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766953.607112:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766954.631229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766954.631245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:3.0:1775766954.631246:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766954.631252:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766954.631270:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2843 00000100:00080000:2.0:1775766954.631288:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766954.631298:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766954.631358:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023742012288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766954.631413:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766954.631419:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766954.631422:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766954.631428:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2843 00000100:00080000:2.0:1775766954.631434:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766954.631439:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766954.631453:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766954.631473:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023742012416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766954.631491:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742012288:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775766954.631495:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766954.631516:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766954.631523:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766954.631543:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2843 00000100:00080000:2.0:1775766954.631548:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766954.631557:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766954.631581:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766954.631614:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023742012544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766954.631633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766954.631639:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766954.631642:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775766954.631652:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775766954.631662:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2843 00000100:00080000:2.0:1775766954.631673:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766954.631681:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766954.631693:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766954.631712:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766954/real 1775766954] req@ffff90c2034bdf80 x1862023742012288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766954.631734:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023742012672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766954.631819:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023742012288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766954.631841:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766954.631849:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766954.631856:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742012288:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766954.631961:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766954.631990:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766954.632004:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766954/real 1775766954] req@ffff90c2034bd180 x1862023742012416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766954.632017:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023742012416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766954.632043:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766954.632047:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766954.632051:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012416:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766954.632058:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012544:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766954.632078:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766954.632136:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766954/real 1775766954] req@ffff90c2034bc380 x1862023742012544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766954.632149:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023742012544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766954.632173:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766954.632177:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766954.632180:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766954.632186:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012672:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766954.632206:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766954.632216:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766954/real 1775766954] req@ffff90c2034bce00 x1862023742012672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766979 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766954.632226:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023742012672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766979 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766954.632235:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766954.632238:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766954.632241:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012672:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766955.655200:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766956.679096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766957.703150:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766958.727076:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766959.750988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766959.750998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766959.751003:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766959.751012:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2848 00000100:00080000:2.0:1775766959.751024:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766959.751050:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766959.751106:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742012800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766959.751132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766959.751135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766959.751138:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766959.751141:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2848 00000100:00080000:2.0:1775766959.751144:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766959.751147:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766959.751155:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766959.751166:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742012928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766959.751173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766959.751176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766959.751177:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766959.751179:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2848 00000100:00080000:2.0:1775766959.751182:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766959.751184:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766959.751188:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766959.751197:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742012800:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766959.751201:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023742013056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766959.751224:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766959.751226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766959.751227:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766959.751230:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2848 00000100:00080000:2.0:1775766959.751232:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766959.751234:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766959.751238:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766959.751248:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023742013184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766959.751330:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766959.751352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766959.751399:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766959.751432:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013184:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766960.775110:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766961.799120:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766962.823135:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775766962.823184:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775766962.823220:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775766962.823234:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766962.823269:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766962.823275:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775766962.823281:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775766962.823358:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766959/real 1775766962] req@ffff90c2034bf480 x1862023742013056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766962.823387:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023742013056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766962.823398:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766962.823403:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766962.823408:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766962.823416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766959/real 1775766962] req@ffff90c2034be300 x1862023742013184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766962.823429:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023742013184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766962.823438:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766962.823442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766962.823446:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013184:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775766962.823453:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766959/real 1775766962] req@ffff90c2034bca80 x1862023742012800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766962.823466:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742012800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766962.823474:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766962.823478:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766962.823481:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742012800:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766962.823488:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766959/real 1775766962] req@ffff90c2034bea00 x1862023742012928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766984 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766962.823498:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742012928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766984 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766962.823505:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766962.823508:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766962.823512:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742012928:192.168.202.155@tcp:38:kworker.0 00010000:00010000:3.0:1775766963.847100:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766964.871081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766964.871096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766964.871104:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766964.871119:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2853 00000100:00080000:0.0:1775766964.871137:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766964.871149:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766964.871218:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742013312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775766964.871256:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766964.871282:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766964.871292:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766964.871296:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766964.871303:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2853 00000100:00080000:0.0:1775766964.871310:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766964.871317:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766964.871335:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766964.871346:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742013312:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766964.871392:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742013440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766964.871415:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766964.871421:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766964.871425:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766964.871433:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2853 00000800:00000100:2.0:1775766964.871440:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775766964.871447:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766964.871463:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766964.871478:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766964.871506:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742013568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766964.871510:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766964/real 1775766964] req@ffff90c203638700 x1862023742013312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766964.871567:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766964.871575:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766964.871579:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775766964.871586:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742013312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766964.871593:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2853 00000100:00080000:0.0:1775766964.871605:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766964.871616:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775766964.871637:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1775766964.871639:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775766964.871654:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766964.871662:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742013312:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766964.871679:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742013696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766964.871707:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766964.871754:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766964.871777:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766964/real 1775766964] req@ffff90c20363aa00 x1862023742013440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766964.871795:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742013440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766964.871807:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766964.871812:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766964.871816:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766964.871826:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766964.871857:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766964.871907:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766964/real 1775766964] req@ffff90c203639180 x1862023742013568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766964.871923:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742013568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766964.871932:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766964.871935:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766964.871939:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766964.871960:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013696:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766964.872009:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766964.872024:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766964/real 1775766964] req@ffff90c203638000 x1862023742013696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766964.872038:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742013696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766989 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766964.872050:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766964.872054:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766964.872057:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013696:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766965.895338:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766966.919058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766967.943050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766968.967044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766969.990968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766969.990980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766969.990987:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766969.991000:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2858 00010000:00010000:3.0:1775766969.991015:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766969.991016:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766969.991026:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775766969.991076:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742013824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766969.991107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766969.991111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766969.991114:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766969.991120:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2858 00000100:00080000:0.0:1775766969.991125:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766969.991130:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766969.991142:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742013824:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775766969.991143:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766969.991163:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742013952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766969.991177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766969.991181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766969.991184:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766969.991189:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2858 00000100:00080000:0.0:1775766969.991194:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766969.991198:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766969.991204:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775766969.991216:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766969.991220:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742014080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766969.991233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766969.991237:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766969.991240:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766969.991245:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2858 00000100:00080000:0.0:1775766969.991249:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766969.991254:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766969.991261:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775766969.991264:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766969/real 1775766969] req@ffff90c203639500 x1862023742013824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766969.991278:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742014208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766969.991292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742013824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766969.991302:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766969.991307:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766969.991311:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742013824:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766969.991334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766969.991350:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766969.991359:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766969/real 1775766969] req@ffff90c20363b100 x1862023742013952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766969.991367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742013952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766969.991374:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766969.991377:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766969.991381:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742013952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766969.991387:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014080:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766969.991406:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766969.991418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766969/real 1775766969] req@ffff90c20363bb80 x1862023742014080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766969.991432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742014080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766969.991446:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766969.991449:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766969.991451:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766969.991456:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014208:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766969.991472:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766969.991479:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766969/real 1775766969] req@ffff90c20363a680 x1862023742014208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766969.991487:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742014208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775766994 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766969.991495:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766969.991497:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766969.991499:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014208:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766971.015045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766972.039022:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766973.063042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766974.087047:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766975.110984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766975.110994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775766975.110999:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766975.111011:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2863 00000100:00080000:0.0:1775766975.111024:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766975.111032:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775766975.111034:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775766975.111078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742014336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766975.111103:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766975.111107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775766975.111109:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766975.111112:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2863 00000100:00080000:0.0:1775766975.111115:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766975.111118:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766975.111127:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766975.111139:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742014464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775766975.111140:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742014336:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775766975.111154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766975.111157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775766975.111158:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766975.111160:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2863 00000100:00080000:0.0:1775766975.111162:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766975.111164:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766975.111167:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766975.111175:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742014592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775766975.111182:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775766975.111184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775766975.111185:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775766975.111187:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2863 00000100:00080000:0.0:1775766975.111191:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775766975.111193:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775766975.111197:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775766975.111204:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742014720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775766975.111221:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766975.111260:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766975/real 1775766975] req@ffff90c203638e00 x1862023742014336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766975.111277:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742014336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766975.111287:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766975.111290:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766975.111293:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742014336:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766975.111316:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014464:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766975.111336:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766975.111349:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766975/real 1775766975] req@ffff90c20363b480 x1862023742014464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766975.111362:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742014464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766975.111372:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766975.111376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766975.111379:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766975.111385:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014592:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775766975.111404:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766975.111412:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766975/real 1775766975] req@ffff90c203639f80 x1862023742014592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766975.111420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742014592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766975.111426:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766975.111428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766975.111430:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775766975.111434:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014720:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775766975.111444:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766975.111449:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766975/real 1775766975] req@ffff90c20363b800 x1862023742014720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775766975.111468:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742014720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775766975.111475:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775766975.111477:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775766975.111479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014720:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766976.135040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766977.159056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775766978.183050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766979.207033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775766980.231102:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766980.231121:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775766980.231130:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766980.231151:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2868 00000100:00080000:2.0:1775766980.231172:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766980.231187:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775766980.231262:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd500 x1862023742014848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766980.231336:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766980.231344:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775766980.231348:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766980.231356:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2868 00000100:00080000:2.0:1775766980.231364:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766980.231372:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766980.231401:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775766980.231424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742014848:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775766980.231427:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bfb80 x1862023742014976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766980.231461:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766980.231465:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775766980.231467:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766980.231472:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2868 00000100:00080000:2.0:1775766980.231477:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766980.231481:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766980.231487:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775766980.231505:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742015104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775766980.231516:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775766980.231521:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775766980.231523:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775766980.231529:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2868 00000100:00080000:2.0:1775766980.231533:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775766980.231536:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775766980.231542:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775766980.231552:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775766980.231558:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742015232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766980.231632:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766980/real 1775766980] req@ffff90c2034bd500 x1862023742014848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766980.231674:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd500 x1862023742014848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1775766980.231678:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775766980.231716:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766980.231728:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766980.231738:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742014848:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766980.231838:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766980.231928:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766980.231966:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766980/real 1775766980] req@ffff90c2034bfb80 x1862023742014976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766980.231990:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bfb80 x1862023742014976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766980.232004:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766980.232009:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766980.232015:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742014976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766980.232025:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766980.232056:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766980.232071:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766980/real 1775766980] req@ffff90c2034bea00 x1862023742015104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766980.232090:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742015104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766980.232101:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766980.232106:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766980.232111:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766980.232120:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015232:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766980.232163:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766980.232179:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766980/real 1775766980] req@ffff90c2034bca80 x1862023742015232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766980.232215:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742015232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766980.232233:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766980.232240:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766980.232245:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015232:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766981.255015:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766982.279077:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766983.303108:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766984.327080:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766985.350964:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766985.350975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766985.350981:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766985.350992:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2873 00000100:00080000:1.0:1775766985.351004:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766985.351014:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766985.351016:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766985.351074:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742015360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766985.351114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766985.351118:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766985.351120:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766985.351125:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2873 00000100:00080000:1.0:1775766985.351130:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766985.351134:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766985.351148:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766985.351162:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742015488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766985.351177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766985.351184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766985.351188:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766985.351189:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742015360:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766985.351198:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2873 00000100:00080000:1.0:1775766985.351204:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766985.351208:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766985.351212:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766985.351225:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742015616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766985.351232:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766985.351235:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766985.351236:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766985.351240:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2873 00000100:00080000:1.0:1775766985.351244:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766985.351246:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766985.351250:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766985.351257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742015744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766985.351277:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766985.351318:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766985/real 1775766985] req@ffff90c203494380 x1862023742015360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766985.351342:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742015360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766985.351353:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766985.351357:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766985.351361:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742015360:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766985.351384:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015488:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766985.351402:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766985.351413:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766985/real 1775766985] req@ffff90c203495c00 x1862023742015488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766985.351424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742015488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766985.351432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766985.351435:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766985.351438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766985.351443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015616:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766985.351457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766985.351465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766985/real 1775766985] req@ffff90c203497100 x1862023742015616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766985.351475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742015616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766985.351496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766985.351499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766985.351502:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766985.351508:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015744:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766985.351523:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766985.351531:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766985/real 1775766985] req@ffff90c203495880 x1862023742015744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766985.351541:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742015744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766985.351550:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766985.351553:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766985.351555:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742015744:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766986.375100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766987.399075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766988.423042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766989.447052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766990.470969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766990.470980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766990.470985:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766990.470996:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2878 00000100:00080000:1.0:1775766990.471011:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766990.471018:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766990.471028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766990.471067:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742015872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766990.471096:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766990.471099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766990.471100:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766990.471103:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2878 00000100:00080000:1.0:1775766990.471106:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766990.471109:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766990.471120:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766990.471129:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742016000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766990.471137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766990.471140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766990.471143:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775766990.471148:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742015872:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775766990.471148:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2878 00000100:00080000:1.0:1775766990.471156:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766990.471167:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766990.471171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766990.471184:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742016128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766990.471190:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766990.471194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766990.471196:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766990.471199:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2878 00000100:00080000:1.0:1775766990.471203:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766990.471205:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766990.471209:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766990.471238:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742016256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775766990.471247:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766990.471284:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766990/real 1775766990] req@ffff90c203496d80 x1862023742015872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766990.471299:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742015872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766990.471307:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766990.471311:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766990.471314:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742015872:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766990.471335:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766990.471350:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766990.471375:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766990/real 1775766990] req@ffff90c203497b80 x1862023742016000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766990.471384:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742016000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766990.471389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766990.471392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766990.471394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766990.471398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766990.471409:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766990.471414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766990/real 1775766990] req@ffff90c203496300 x1862023742016128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766990.471422:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742016128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766990.471434:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766990.471436:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766990.471438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766990.471442:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016256:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766990.471452:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766990.471457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766990/real 1775766990] req@ffff90c203496a00 x1862023742016256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766990.471464:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742016256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766990.471471:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766990.471473:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766990.471475:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016256:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766991.495063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766992.519077:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766993.543092:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766994.567064:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775766995.591037:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766995.591047:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775766995.591052:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766995.591063:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2884 00000100:00080000:1.0:1775766995.591076:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766995.591084:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775766995.591119:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775766995.591124:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742016384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766995.591163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766995.591167:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775766995.591168:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766995.591171:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2884 00000100:00080000:1.0:1775766995.591175:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766995.591178:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766995.591186:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766995.591196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742016512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766995.591204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766995.591218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775766995.591220:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766995.591223:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2884 00000100:00080000:1.0:1775766995.591227:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766995.591230:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766995.591235:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775766995.591245:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742016640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775766995.591256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775766995.591260:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775766995.591262:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775766995.591267:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2884 00000100:00080000:1.0:1775766995.591271:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775766995.591275:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775766995.591282:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775766995.591293:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742016768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775766995.591340:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742016384:192.168.202.155@tcp:250:kworker.0 00000800:00000100:0.0:1775766995.591444:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766995.591505:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766995/real 1775766995] req@ffff90c203496680 x1862023742016384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766995.591528:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742016384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766995.591543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766995.591548:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766995.591553:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742016384:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775766995.591561:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766995.591583:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766995.591595:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766995/real 1775766995] req@ffff90c203494a80 x1862023742016512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766995.591607:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742016512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766995.591624:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766995.591628:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766995.591631:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766995.591637:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775766995.591721:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766995.591741:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766995/real 1775766995] req@ffff90c203494700 x1862023742016640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766995.591777:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742016640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766995.591791:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766995.591797:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766995.591802:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775766995.591813:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016768:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775766995.591873:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775766995.591887:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775766995/real 1775766995] req@ffff90c203495500 x1862023742016768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775766995.591981:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742016768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775766995.591992:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775766995.591995:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775766995.591998:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742016768:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775766996.615022:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766997.639172:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766998.663039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775766999.687199:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767000.711010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775767000.711010:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767000.711020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767000.711026:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767000.711037:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2889 00000100:00080000:0.0:1775767000.711048:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767000.711055:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767000.711102:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742016896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767000.711150:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767000.711154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767000.711156:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767000.711159:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2889 00000100:00080000:0.0:1775767000.711163:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767000.711167:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767000.711179:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767000.711191:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742016896:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767000.711197:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742017024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767000.711208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767000.711211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767000.711212:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767000.711215:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2889 00000100:00080000:0.0:1775767000.711218:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767000.711222:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767000.711227:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767000.711238:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742017152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767000.711246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767000.711249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767000.711250:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767000.711253:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2889 00000100:00080000:0.0:1775767000.711256:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767000.711259:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767000.711263:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767000.711272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742017280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767000.711280:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767000.711330:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767000/real 1775767000] req@ffff90c203638380 x1862023742016896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767000.711364:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742016896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767000.711375:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767000.711379:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767000.711383:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742016896:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767000.711406:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017024:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767000.711424:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767000.711434:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767000/real 1775767000] req@ffff90c20363b800 x1862023742017024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767000.711444:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742017024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767000.711451:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767000.711454:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767000.711456:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767000.711461:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017152:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767000.711491:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767000.711500:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767000/real 1775767000] req@ffff90c203639f80 x1862023742017152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767000.711520:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742017152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767000.711535:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767000.711539:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767000.711543:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767000.711550:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017280:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767000.711571:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767000.711580:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767000/real 1775767000] req@ffff90c20363b480 x1862023742017280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767025 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767000.711588:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742017280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767025 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767000.711596:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767000.711599:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767000.711626:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017280:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767001.735118:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767002.759122:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767003.783162:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767004.807105:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767005.831020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767005.831033:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767005.831039:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767005.831050:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2894 00000100:00080000:1.0:1775767005.831065:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767005.831073:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767005.831095:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767005.831130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742017408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767005.831153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767005.831156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767005.831158:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767005.831161:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2894 00000100:00080000:1.0:1775767005.831163:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767005.831166:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767005.831176:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767005.831186:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742017536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767005.831195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767005.831199:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767005.831200:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742017408:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767005.831201:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767005.831206:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2894 00000100:00080000:1.0:1775767005.831211:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767005.831213:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767005.831216:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767005.831224:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742017664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767005.831230:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767005.831233:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767005.831235:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767005.831238:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2894 00000100:00080000:1.0:1775767005.831241:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767005.831242:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767005.831246:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767005.831252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742017792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767005.831302:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767005.831348:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767005/real 1775767005] req@ffff90c203495180 x1862023742017408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767005.831370:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742017408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767005.831383:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767005.831388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767005.831393:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742017408:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767005.831423:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017536:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767005.831447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767005.831460:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767005/real 1775767005] req@ffff90c203495500 x1862023742017536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767005.831473:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742017536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767005.831483:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767005.831486:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767005.831489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767005.831495:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767005.831517:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767005.831527:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767005/real 1775767005] req@ffff90c203494700 x1862023742017664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767005.831540:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742017664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767005.831564:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767005.831568:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767005.831572:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767005.831579:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017792:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767005.831603:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767005.831614:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767005/real 1775767005] req@ffff90c203494a80 x1862023742017792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767005.831629:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742017792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767030 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767005.831649:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767005.831652:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767005.831656:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742017792:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767006.855156:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767007.879178:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767008.903220:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767009.927142:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767010.951076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767010.951090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767010.951097:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767010.951110:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2899 00000100:00080000:1.0:1775767010.951126:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767010.951136:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767010.951207:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742017920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767010.951208:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767010.951279:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767010.951284:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767010.951286:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767010.951291:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2899 00000100:00080000:1.0:1775767010.951295:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767010.951300:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767010.951326:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767010.951344:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742018048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767010.951345:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742017920:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767010.951369:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767010.951383:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767010.951384:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767010.951388:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2899 00000100:00080000:1.0:1775767010.951391:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767010.951395:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767010.951400:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767010.951412:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742018176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767010.951423:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767010.951426:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767010.951428:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767010.951432:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2899 00000100:00080000:1.0:1775767010.951436:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767010.951440:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767010.951445:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767010.951457:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742018304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767010.951475:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767010.951598:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767010/real 1775767010] req@ffff90c203496680 x1862023742017920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767010.951636:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742017920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767010.951649:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767010.951654:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767010.951658:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742017920:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767010.951689:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018048:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767010.951717:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767010.951730:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767010/real 1775767010] req@ffff90c203496a00 x1862023742018048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767010.951742:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742018048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767010.951758:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767010.951761:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767010.951765:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767010.951770:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018176:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767010.951785:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767010.951793:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767010/real 1775767010] req@ffff90c203496300 x1862023742018176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767010.951804:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742018176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767010.951812:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767010.951815:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767010.951818:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767010.951823:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018304:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767010.951838:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767010.951845:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767010/real 1775767010] req@ffff90c203497b80 x1862023742018304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767010.951874:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742018304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767035 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767010.951916:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767010.951920:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767010.951924:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018304:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767011.975097:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767012.999230:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767014.023150:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767015.047114:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767016.070994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767016.071008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767016.071015:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767016.071028:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2904 00000100:00080000:1.0:1775767016.071044:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767016.071054:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767016.071123:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767016.071138:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742018432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767016.071192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767016.071197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767016.071199:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767016.071204:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2904 00000100:00080000:1.0:1775767016.071208:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767016.071212:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767016.071244:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767016.071256:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742018560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767016.071268:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742018432:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767016.071269:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767016.071279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767016.071282:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767016.071287:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2904 00000100:00080000:1.0:1775767016.071290:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767016.071293:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767016.071297:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767016.071309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742018688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767016.071318:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767016.071322:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767016.071323:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767016.071328:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2904 00000100:00080000:1.0:1775767016.071332:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767016.071334:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767016.071338:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767016.071347:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742018816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767016.071361:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767016.071409:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767016/real 1775767016] req@ffff90c203496d80 x1862023742018432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767016.071428:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742018432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767016.071439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767016.071461:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767016.071466:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742018432:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767016.071493:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018560:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767016.071515:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767016.071527:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767016/real 1775767016] req@ffff90c203495880 x1862023742018560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767016.071537:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742018560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767016.071545:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767016.071548:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767016.071551:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767016.071556:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767016.071568:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767016.071576:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767016/real 1775767016] req@ffff90c203497100 x1862023742018688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767016.071586:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742018688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767016.071592:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767016.071594:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767016.071596:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767016.071601:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018816:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767016.071613:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767016.071619:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767016/real 1775767016] req@ffff90c203495c00 x1862023742018816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767016.071628:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742018816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767016.071636:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767016.071639:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767016.071641:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742018816:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767017.095178:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767018.119068:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767019.143165:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767020.167051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767021.191085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767021.191098:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:3.0:1775767021.191101:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767021.191105:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767021.191119:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2909 00000100:00080000:2.0:1775767021.191134:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767021.191143:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767021.191203:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023742018944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767021.191234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767021.191238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767021.191240:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767021.191244:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2909 00000100:00080000:2.0:1775767021.191247:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767021.191250:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767021.191261:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767021.191272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023742019072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767021.191281:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767021.191285:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767021.191286:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767021.191289:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2909 00000100:00080000:2.0:1775767021.191292:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767021.191295:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767021.191299:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767021.191309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023742019200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767021.191317:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767021.191320:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767021.191322:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767021.191327:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2909 00000100:00080000:2.0:1775767021.191330:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767021.191336:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767021.191338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742018944:192.168.202.155@tcp:250:kworker.0 00000008:00100000:2.0:1775767021.191341:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767021.191355:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023742019328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767021.191460:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767021.191513:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767021/real 1775767021] req@ffff90c2034be300 x1862023742018944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767021.191543:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023742018944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767021.191559:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767021.191566:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767021.191572:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742018944:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767021.191588:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767021.191614:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767021.191627:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767021/real 1775767021] req@ffff90c2034bf480 x1862023742019072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767021.191638:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023742019072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767021.191646:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767021.191649:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767021.191652:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767021.191657:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767021.191670:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767021.191678:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767021/real 1775767021] req@ffff90c2034bce00 x1862023742019200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767021.191687:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023742019200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767021.191693:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767021.191695:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767021.191698:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767021.191731:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019328:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767021.191756:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767021.191770:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767021/real 1775767021] req@ffff90c2034bc380 x1862023742019328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767021.191788:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023742019328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767021.191818:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767021.191822:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767021.191826:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019328:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767022.215074:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767023.239083:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767024.263110:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767025.287090:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767026.311036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767026.311049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767026.311054:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767026.311066:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2914 00000100:00080000:0.0:1775767026.311080:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767026.311088:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767026.311136:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742019456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767026.311137:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767026.311173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767026.311176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767026.311178:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767026.311181:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2914 00000100:00080000:0.0:1775767026.311183:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767026.311187:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767026.311196:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767026.311207:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742019456:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767026.311208:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742019584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767026.311229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767026.311231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767026.311233:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767026.311236:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2914 00000100:00080000:0.0:1775767026.311238:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767026.311241:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767026.311244:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767026.311252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742019712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767026.311258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767026.311261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767026.311262:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767026.311265:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2914 00000100:00080000:0.0:1775767026.311268:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767026.311270:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767026.311273:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767026.311281:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742019840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767026.311347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767026.311379:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767026.311393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019840:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767027.335097:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767028.359098:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767029.383051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767029.383168:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775767029.383211:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767029.383229:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767029.383265:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767029.383274:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767029.383283:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767029.383330:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767026/real 1775767029] req@ffff90c20363a680 x1862023742019584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767029.383355:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742019584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767029.383368:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767029.383375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767029.383380:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767029.383390:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767026/real 1775767029] req@ffff90c20363bb80 x1862023742019712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767029.383402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742019712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767029.383410:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767029.383414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767029.383418:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767029.383426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767026/real 1775767029] req@ffff90c20363b100 x1862023742019840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767029.383437:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742019840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767029.383448:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767029.383452:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767029.383455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742019840:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767029.383462:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767026/real 1775767029] req@ffff90c203638e00 x1862023742019456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767029.383475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742019456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767029.383482:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767029.383485:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767029.383488:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742019456:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767030.407096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767031.430987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767031.431003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767031.431009:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767031.431025:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2919 00010000:00010000:3.0:1775767031.431032:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767031.431042:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767031.431052:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767031.431102:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742019968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767031.431133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767031.431137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767031.431139:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767031.431144:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2919 00000100:00080000:0.0:1775767031.431148:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767031.431163:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767031.431175:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767031.431183:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742019968:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767031.431203:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742020096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767031.431216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767031.431220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767031.431222:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767031.431226:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2919 00000100:00080000:0.0:1775767031.431230:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767031.431234:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767031.431239:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767031.431251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742020224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767031.431261:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767031.431265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767031.431268:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767031.431272:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2919 00000100:00080000:0.0:1775767031.431276:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767031.431313:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767031.431318:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:0.0:1775767031.431322:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767031.431337:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742020352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767031.431380:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767031/real 1775767031] req@ffff90c203639500 x1862023742019968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767031.431406:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742019968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767031.431420:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767031.431425:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767031.431429:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742019968:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767031.431471:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767031.431502:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767031.431517:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767031/real 1775767031] req@ffff90c203638000 x1862023742020096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767031.431531:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742020096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767031.431541:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767031.431546:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767031.431549:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767031.431556:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767031.431579:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767031.431592:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767031/real 1775767031] req@ffff90c203639180 x1862023742020224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767031.431607:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742020224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767031.431624:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767031.431629:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767031.431633:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767031.431640:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020352:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767031.431663:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767031.431675:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767031/real 1775767031] req@ffff90c20363aa00 x1862023742020352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767031.431688:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742020352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767031.431707:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767031.431712:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767031.431716:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020352:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767032.455101:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767033.479116:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767034.503161:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767035.527118:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767036.550963:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767036.550974:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767036.550979:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767036.550991:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2925 00000100:00080000:0.0:1775767036.551004:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767036.551011:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767036.551057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767036.551060:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742020480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767036.551086:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767036.551090:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767036.551091:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767036.551094:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2925 00000100:00080000:0.0:1775767036.551097:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767036.551100:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767036.551110:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767036.551120:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742020480:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767036.551123:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742020608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767036.551134:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767036.551137:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767036.551138:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767036.551141:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2925 00000100:00080000:0.0:1775767036.551144:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767036.551146:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767036.551149:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767036.551156:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742020736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767036.551162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767036.551165:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767036.551166:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767036.551170:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2925 00000100:00080000:0.0:1775767036.551172:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767036.551174:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767036.551177:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767036.551187:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742020864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767036.551199:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767036.551252:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767036/real 1775767036] req@ffff90c203638700 x1862023742020480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767036.551270:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742020480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767036.551278:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767036.551282:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767036.551285:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742020480:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767036.551308:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767036.551324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767036.551332:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767036/real 1775767036] req@ffff90c20363aa00 x1862023742020608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767036.551340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742020608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767036.551346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767036.551348:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767036.551350:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767036.551354:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020736:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767036.551363:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767036.551369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767036/real 1775767036] req@ffff90c203639180 x1862023742020736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767036.551376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742020736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767036.551381:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767036.551383:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767036.551384:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767036.551388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020864:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767036.551397:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767036.551402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767036/real 1775767036] req@ffff90c203638000 x1862023742020864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767036.551410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742020864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767036.551417:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767036.551419:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767036.551421:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742020864:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767037.575052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767038.599011:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767039.622991:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767040.647056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767041.671009:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767041.671021:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767041.671027:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767041.671038:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2930 00000100:00080000:0.0:1775767041.671052:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767041.671061:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767041.671091:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767041.671119:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742020992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767041.671141:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767041.671144:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767041.671146:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767041.671149:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2930 00000100:00080000:0.0:1775767041.671152:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767041.671154:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767041.671165:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767041.671205:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742020992:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767041.671209:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742021120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767041.671225:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767041.671228:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767041.671230:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767041.671236:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2930 00000100:00080000:0.0:1775767041.671239:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767041.671243:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767041.671248:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767041.671261:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742021248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767041.671268:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767041.671271:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767041.671272:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767041.671275:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2930 00000100:00080000:0.0:1775767041.671279:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767041.671281:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767041.671284:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767041.671292:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767041.671295:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742021376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767041.671335:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767041/real 1775767041] req@ffff90c203639500 x1862023742020992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767041.671352:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742020992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767041.671361:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767041.671364:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767041.671368:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742020992:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767041.671391:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767041.671405:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767041.671413:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767041/real 1775767041] req@ffff90c203638e00 x1862023742021120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767041.671421:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742021120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767041.671431:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767041.671434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767041.671436:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767041.671440:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767041.671449:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767041.671454:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767041/real 1775767041] req@ffff90c20363b100 x1862023742021248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767041.671461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742021248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767041.671466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767041.671468:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767041.671470:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767041.671473:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021376:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767041.671483:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767041.671488:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767041/real 1775767041] req@ffff90c20363bb80 x1862023742021376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767066 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767041.671494:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742021376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767066 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767041.671510:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767041.671512:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767041.671515:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021376:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767042.695093:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767043.719102:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767044.743056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767045.767099:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767046.791028:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767046.791040:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767046.791046:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767046.791058:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2935 00000100:00080000:0.0:1775767046.791071:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767046.791079:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767046.791112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767046.791145:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742021504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767046.791172:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767046.791176:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767046.791177:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767046.791181:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2935 00000100:00080000:0.0:1775767046.791184:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767046.791186:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767046.791195:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767046.791207:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742021632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767046.791216:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767046.791219:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767046.791220:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767046.791223:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2935 00000100:00080000:0.0:1775767046.791225:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767046.791227:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767046.791230:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767046.791239:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742021760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767046.791246:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767046.791250:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767046.791251:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767046.791255:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2935 00000100:00080000:0.0:1775767046.791258:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767046.791261:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767046.791264:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767046.791272:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742021888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767046.791282:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742021504:192.168.202.155@tcp:250:kworker.0 00000800:00000100:2.0:1775767046.791353:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767046.791419:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767046/real 1775767046] req@ffff90c20363a680 x1862023742021504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767046.791438:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742021504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767046.791453:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767046.791459:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767046.791465:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742021504:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767046.791476:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767046.791507:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767046.791524:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767046/real 1775767046] req@ffff90c20363b480 x1862023742021632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767046.791543:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742021632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767046.791551:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767046.791554:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767046.791558:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767046.791563:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767046.791583:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767046.791593:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767046/real 1775767046] req@ffff90c203639f80 x1862023742021760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767046.791603:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742021760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767046.791611:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767046.791613:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767046.791616:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767046.791645:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021888:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767046.791660:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767046.791668:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767046/real 1775767046] req@ffff90c20363b800 x1862023742021888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767071 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767046.791677:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742021888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767071 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767046.791687:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767046.791689:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767046.791692:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742021888:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767047.815086:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767048.839073:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767049.863115:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767050.887192:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767051.911015:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767051.911026:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767051.911032:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767051.911042:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2940 00000100:00080000:0.0:1775767051.911057:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767051.911064:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767051.911139:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767051.911147:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742022016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767051.911177:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767051.911180:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767051.911182:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767051.911185:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2940 00000100:00080000:0.0:1775767051.911188:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767051.911190:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767051.911200:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767051.911214:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742022144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767051.911223:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767051.911226:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767051.911228:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767051.911232:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2940 00000100:00080000:0.0:1775767051.911238:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767051.911240:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742022016:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767051.911246:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767051.911256:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767051.911266:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742022272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767051.911273:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767051.911276:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767051.911278:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767051.911282:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2940 00000100:00080000:0.0:1775767051.911284:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767051.911287:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767051.911291:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767051.911300:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742022400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767051.911332:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767051.911389:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767051/real 1775767051] req@ffff90c203638380 x1862023742022016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767051.911418:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742022016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767051.911431:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767051.911435:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767051.911440:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742022016:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767051.911464:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767051.911496:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767051.911511:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767051/real 1775767051] req@ffff90c20363b800 x1862023742022144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767051.911520:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742022144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767051.911526:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767051.911529:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767051.911531:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767051.911535:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022272:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767051.911548:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767051.911554:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767051/real 1775767051] req@ffff90c203639f80 x1862023742022272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767051.911562:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742022272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767051.911567:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767051.911569:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767051.911570:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767051.911575:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022400:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767051.911587:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767051.911592:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767051/real 1775767051] req@ffff90c20363b480 x1862023742022400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767051.911616:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742022400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767076 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767051.911624:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767051.911626:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767051.911628:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022400:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767052.935032:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767053.959149:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767054.983080:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767056.007071:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767057.030948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767057.030959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767057.030964:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767057.030974:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2945 00000100:00080000:0.0:1775767057.030985:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767057.030993:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767057.031008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767057.031042:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742022528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767057.031069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767057.031073:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767057.031075:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767057.031079:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2945 00000100:00080000:0.0:1775767057.031084:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767057.031089:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767057.031099:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767057.031103:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742022528:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767057.031114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742022656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767057.031128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767057.031132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767057.031134:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767057.031138:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2945 00000100:00080000:0.0:1775767057.031142:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767057.031146:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767057.031151:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767057.031158:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767057.031164:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742022784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767057.031175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767057.031178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767057.031180:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767057.031185:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2945 00000100:00080000:0.0:1775767057.031189:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767057.031191:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767057/real 1775767057] req@ffff90c20363a680 x1862023742022528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767057.031197:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767057.031207:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767057.031211:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742022528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767057.031220:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767057.031226:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767057.031227:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742022912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767057.031230:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742022528:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767057.031255:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767057.031271:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767057.031280:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767057/real 1775767057] req@ffff90c20363bb80 x1862023742022656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767057.031288:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742022656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767057.031294:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767057.031296:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767057.031298:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767057.031302:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767057.031311:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767057.031316:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767057/real 1775767057] req@ffff90c20363b100 x1862023742022784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767057.031323:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742022784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767057.031328:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767057.031330:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767057.031332:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767057.031341:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022912:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767057.031352:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767057.031357:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767057/real 1775767057] req@ffff90c203638e00 x1862023742022912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767057.031365:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742022912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767057.031376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767057.031378:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767057.031380:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742022912:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767058.055088:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767059.079110:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767060.103069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767061.127141:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767062.151126:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767062.151196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767062.151205:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767062.151211:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767062.151225:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2950 00000100:00080000:0.0:1775767062.151240:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767062.151248:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767062.151312:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742023040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767062.151345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767062.151349:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767062.151351:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767062.151356:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2950 00000100:00080000:0.0:1775767062.151360:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767062.151364:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767062.151376:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767062.151388:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742023168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767062.151399:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767062.151403:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767062.151405:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767062.151410:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2950 00000100:00080000:0.0:1775767062.151413:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767062.151416:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767062.151422:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767062.151430:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742023296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767062.151440:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767062.151445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767062.151446:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767062.151450:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2950 00000100:00080000:0.0:1775767062.151453:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767062.151457:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767062.151462:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767062.151471:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742023424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767062.151542:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742023040:192.168.202.155@tcp:250:kworker.0 00000800:00000100:2.0:1775767062.151619:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767062.151665:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767062/real 1775767062] req@ffff90c203639500 x1862023742023040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767062.151686:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742023040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767062.151697:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767062.151701:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767062.151706:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742023040:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767062.151713:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767062.151733:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767062.151744:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767062/real 1775767062] req@ffff90c203638000 x1862023742023168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767062.151756:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742023168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767062.151765:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767062.151769:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767062.151772:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767062.151780:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767062.151796:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767062.151806:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767062/real 1775767062] req@ffff90c203639180 x1862023742023296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767062.151818:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742023296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767062.151826:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767062.151830:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767062.151833:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767062.151839:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023424:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767062.151855:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767062.151864:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767062/real 1775767062] req@ffff90c20363aa00 x1862023742023424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767062.151898:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742023424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767062.151909:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767062.151912:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767062.151916:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023424:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767063.175033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767064.199064:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767065.223111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767066.247148:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767067.271034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767067.271047:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767067.271054:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767067.271067:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2955 00000100:00080000:1.0:1775767067.271079:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767067.271087:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767067.271124:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767067.271128:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742023552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767067.271153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767067.271162:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767067.271163:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767067.271167:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2955 00000100:00080000:1.0:1775767067.271170:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767067.271176:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767067.271186:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767067.271196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742023680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767067.271212:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767067.271212:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742023552:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767067.271230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767067.271232:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767067.271238:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2955 00000100:00080000:1.0:1775767067.271242:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767067.271247:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767067.271253:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767067.271268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742023808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767067.271280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767067.271283:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767067.271284:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767067.271288:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2955 00000100:00080000:1.0:1775767067.271293:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767067.271296:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767067.271300:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767067.271311:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742023936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767067.271327:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767067.271386:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767067/real 1775767067] req@ffff90c203494380 x1862023742023552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767067.271410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742023552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767067.271423:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767067.271428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767067.271433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742023552:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767067.271464:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767067.271486:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767067.271498:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767067/real 1775767067] req@ffff90c203494e00 x1862023742023680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767067.271511:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742023680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767067.271528:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767067.271532:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767067.271536:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767067.271542:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023808:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767067.271560:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767067.271571:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767067/real 1775767067] req@ffff90c203495f80 x1862023742023808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767067.271583:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742023808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767067.271599:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767067.271602:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767067.271605:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767067.271612:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023936:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767067.271629:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767067.271638:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767067/real 1775767067] req@ffff90c203494000 x1862023742023936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767067.271651:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742023936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767067.271661:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767067.271665:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767067.271668:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742023936:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767068.295125:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767069.319146:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767070.343020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767071.367128:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767072.391013:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767072.391021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767072.391025:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767072.391033:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2960 00000100:00080000:1.0:1775767072.391044:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767072.391052:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775767072.391068:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767072.391094:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742024064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767072.391123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767072.391126:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767072.391128:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767072.391131:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2960 00000100:00080000:1.0:1775767072.391134:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767072.391138:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767072.391147:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767072.391159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742024192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767072.391167:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742024064:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767072.391173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767072.391178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767072.391180:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767072.391186:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2960 00000100:00080000:1.0:1775767072.391191:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767072.391197:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767072.391204:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767072.391218:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742024320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767072.391228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767072.391231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767072.391232:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767072.391235:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2960 00000100:00080000:1.0:1775767072.391239:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767072.391242:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767072.391247:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775767072.391248:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767072.391261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742024448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767072.391299:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767072/real 1775767072] req@ffff90c203497480 x1862023742024064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767072.391317:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742024064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767072.391326:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767072.391330:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767072.391333:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742024064:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767072.391359:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767072.391376:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767072.391384:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767072/real 1775767072] req@ffff90c203494000 x1862023742024192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767072.391393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742024192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767072.391399:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767072.391401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767072.391403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767072.391407:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767072.391418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767072.391423:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767072/real 1775767072] req@ffff90c203495f80 x1862023742024320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767072.391431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742024320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767072.391435:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767072.391437:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767072.391439:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767072.391443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024448:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767072.391454:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767072.391459:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767072/real 1775767072] req@ffff90c203494e00 x1862023742024448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767072.391467:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742024448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767072.391479:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767072.391483:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767072.391486:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024448:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767073.415058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767074.439002:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767075.463098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767076.487023:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767077.511016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767077.511027:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767077.511034:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767077.511048:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2965 00000100:00080000:1.0:1775767077.511063:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767077.511073:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767077.511085:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767077.511130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742024576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767077.511167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767077.511172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767077.511174:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767077.511177:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2965 00000100:00080000:1.0:1775767077.511181:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767077.511185:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767077.511196:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767077.511212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742024704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767077.511222:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767077.511226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767077.511227:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767077.511233:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2965 00000100:00100000:2.0:1775767077.511234:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742024576:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767077.511236:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767077.511243:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767077.511247:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767077.511255:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742024832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767077.511267:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767077.511270:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767077.511272:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767077.511275:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2965 00000100:00080000:1.0:1775767077.511277:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767077.511279:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767077.511284:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767077.511292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742024960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767077.511351:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767077.511417:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767077/real 1775767077] req@ffff90c203494380 x1862023742024576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767077.511434:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742024576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767077.511443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767077.511446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767077.511449:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742024576:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767077.511473:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767077.511489:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767077.511497:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767077/real 1775767077] req@ffff90c203495c00 x1862023742024704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767077.511505:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742024704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767077.511517:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767077.511519:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767077.511521:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767077.511525:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024832:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767077.511535:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767077.511541:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767077/real 1775767077] req@ffff90c203497100 x1862023742024832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767077.511548:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742024832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767077.511554:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767077.511556:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767077.511558:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767077.511561:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024960:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767077.511571:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767077.511577:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767077/real 1775767077] req@ffff90c203495880 x1862023742024960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767077.511584:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742024960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767077.511591:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767077.511593:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767077.511596:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742024960:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767078.535028:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767079.559072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767080.583041:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767081.607050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767082.631021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767082.631033:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767082.631038:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767082.631051:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2971 00000100:00080000:1.0:1775767082.631065:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767082.631078:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767082.631100:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767082.631130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742025088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767082.631160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767082.631163:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767082.631165:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767082.631168:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2971 00000100:00080000:1.0:1775767082.631171:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767082.631173:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767082.631185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767082.631195:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742025216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767082.631205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767082.631207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767082.631209:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767082.631216:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2971 00000100:00100000:2.0:1775767082.631217:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742025088:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767082.631219:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767082.631227:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767082.631236:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767082.631244:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742025344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767082.631251:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767082.631255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767082.631257:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767082.631261:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2971 00000100:00080000:1.0:1775767082.631264:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767082.631266:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767082.631270:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767082.631278:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742025472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767082.631298:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767082.631336:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767082/real 1775767082] req@ffff90c203496d80 x1862023742025088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767082.631351:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742025088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767082.631360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767082.631365:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767082.631369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742025088:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767082.631394:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025216:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767082.631418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767082.631442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767082/real 1775767082] req@ffff90c203497b80 x1862023742025216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767082.631451:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742025216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767082.631457:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767082.631460:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767082.631462:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767082.631466:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767082.631481:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767082.631487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767082/real 1775767082] req@ffff90c203496300 x1862023742025344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767082.631495:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742025344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767082.631500:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767082.631502:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767082.631504:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767082.631508:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025472:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767082.631518:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767082.631524:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767082/real 1775767082] req@ffff90c203496a00 x1862023742025472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767107 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767082.631531:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742025472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767107 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767082.631538:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767082.631539:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767082.631541:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025472:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767083.655084:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767084.679069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767085.703084:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767086.727103:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767087.751035:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767087.751044:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767087.751064:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767087.751074:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2976 00010000:00010000:1.0:1775767087.751083:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767087.751084:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767087.751094:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767087.751130:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742025600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767087.751172:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767087.751176:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767087.751179:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767087.751183:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2976 00000100:00080000:0.0:1775767087.751187:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767087.751192:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767087.751202:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767087.751228:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742025600:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767087.751229:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742025728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767087.751243:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767087.751245:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767087.751246:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767087.751251:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2976 00000100:00080000:0.0:1775767087.751254:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767087.751258:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767087.751264:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767087.751276:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742025856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767087.751288:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767087.751290:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767087.751295:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767087.751297:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767087.751304:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2976 00000100:00080000:0.0:1775767087.751309:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767087.751313:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767087.751325:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767087.751325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767087/real 1775767087] req@ffff90c203638700 x1862023742025600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767087.751342:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742025984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767087.751346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742025600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767087.751366:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767087.751370:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767087.751372:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742025600:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767087.751391:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767087.751406:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767087.751431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767087/real 1775767087] req@ffff90c20363aa00 x1862023742025728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767087.751439:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742025728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767087.751445:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767087.751448:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767087.751459:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767087.751463:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767087.751473:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767087.751479:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767087/real 1775767087] req@ffff90c203639180 x1862023742025856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767087.751486:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742025856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767087.751491:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767087.751493:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767087.751495:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767087.751499:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025984:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767087.751509:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767087.751514:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767087/real 1775767087] req@ffff90c203638000 x1862023742025984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767112 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767087.751521:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742025984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767112 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767087.751528:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767087.751529:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767087.751531:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742025984:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767088.775123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767089.799023:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767090.823062:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767091.847054:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767092.870980:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767092.870988:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767092.870992:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767092.871000:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2981 00000100:00080000:0.0:1775767092.871031:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767092.871037:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767092.871067:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767092.871080:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742026112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767092.871102:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767092.871105:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767092.871106:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767092.871109:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2981 00000100:00080000:0.0:1775767092.871111:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767092.871114:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767092.871122:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767092.871130:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742026240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767092.871136:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767092.871139:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767092.871140:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767092.871142:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2981 00000100:00080000:0.0:1775767092.871144:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767092.871147:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767092.871149:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767092.871155:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742026368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767092.871161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767092.871164:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767092.871165:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767092.871168:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2981 00000100:00080000:0.0:1775767092.871170:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767092.871172:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767092.871176:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767092.871179:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742026112:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767092.871184:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742026496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767092.871276:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767092.871311:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767092.871339:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026496:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767093.895050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767094.919100:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767095.943028:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775767095.943200:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775767095.943247:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767095.943266:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767095.943301:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767095.943311:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767095.943321:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767095.943340:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767092/real 1775767095] req@ffff90c203639500 x1862023742026112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767095.943368:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742026112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767095.943383:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767095.943389:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767095.943394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742026112:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767095.943402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767092/real 1775767095] req@ffff90c203638e00 x1862023742026240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767095.943423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742026240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767095.943432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767095.943436:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767095.943439:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767095.943447:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767092/real 1775767095] req@ffff90c20363b100 x1862023742026368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767095.943460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742026368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767095.943468:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767095.943483:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767095.943487:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767095.943520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767092/real 1775767095] req@ffff90c20363bb80 x1862023742026496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767095.943533:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742026496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767117 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767095.943544:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767095.943549:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767095.943552:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026496:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767096.967060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767097.991124:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767097.991139:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767097.991149:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767097.991155:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767097.991167:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2986 00000100:00080000:0.0:1775767097.991181:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767097.991189:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767097.991239:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742026624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767097.991273:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767097.991280:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767097.991282:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767097.991288:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2986 00000100:00080000:0.0:1775767097.991293:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767097.991297:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767097.991311:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767097.991331:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742026752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767097.991347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742026624:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767097.991354:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767097.991362:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767097.991364:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767097.991372:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2986 00000100:00080000:0.0:1775767097.991377:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767097.991381:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767097.991389:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767097.991406:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742026880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767097.991419:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767097.991425:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767097.991427:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767097.991433:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2986 00000100:00080000:0.0:1775767097.991439:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767097.991445:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767097.991453:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767097.991462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767097.991482:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742027008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767097.991528:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767097/real 1775767097] req@ffff90c20363a680 x1862023742026624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767097.991557:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742026624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767097.991572:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767097.991578:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767097.991637:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742026624:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767097.991683:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767097.991721:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767097.991761:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767097/real 1775767097] req@ffff90c20363b480 x1862023742026752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767097.991787:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742026752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767097.991800:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767097.991807:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767097.991813:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767097.991823:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767097.991855:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767097.991907:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767097/real 1775767097] req@ffff90c203639f80 x1862023742026880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767097.991928:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742026880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767097.991941:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767097.991947:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767097.991952:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742026880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767097.991963:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027008:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767097.991996:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767097.992013:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767097/real 1775767097] req@ffff90c20363b800 x1862023742027008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767097.992031:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742027008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767122 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767097.992046:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767097.992052:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767097.992056:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027008:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767099.015112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767100.039091:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767101.063108:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767102.087053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767103.111013:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767103.111027:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767103.111033:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767103.111046:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2991 00000100:00080000:0.0:1775767103.111063:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767103.111073:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767103.111076:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767103.111129:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742027136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767103.111161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767103.111165:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767103.111167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767103.111173:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2991 00000100:00080000:0.0:1775767103.111178:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767103.111183:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767103.111195:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742027136:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775767103.111196:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767103.111220:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742027264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767103.111234:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767103.111238:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767103.111240:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767103.111246:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2991 00000100:00080000:0.0:1775767103.111250:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767103.111255:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767103.111260:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767103.111270:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767103.111275:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742027392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767103.111287:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767103.111305:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767103.111307:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767103.111313:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2991 00000100:00080000:0.0:1775767103.111318:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767103.111318:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767103/real 1775767103] req@ffff90c203638380 x1862023742027136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767103.111327:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767103.111337:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767103.111343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742027136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767103.111353:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742027520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767103.111371:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767103.111377:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767103.111382:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742027136:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767103.111405:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767103.111420:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767103.111429:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767103/real 1775767103] req@ffff90c20363b800 x1862023742027264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767103.111437:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742027264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767103.111443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767103.111446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767103.111448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767103.111451:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027392:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767103.111461:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767103.111466:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767103/real 1775767103] req@ffff90c203639f80 x1862023742027392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767103.111474:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742027392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767103.111479:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767103.111481:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767103.111482:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767103.111487:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027520:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767103.111498:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767103.111503:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767103/real 1775767103] req@ffff90c20363b480 x1862023742027520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767103.111510:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742027520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767103.111517:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767103.111519:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767103.111521:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027520:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767104.135062:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767105.159008:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767106.183017:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767107.207006:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767108.230999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767108.231011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767108.231018:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767108.231031:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 2996 00000100:00080000:1.0:1775767108.231046:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767108.231054:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767108.231106:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742027648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767108.231113:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767108.231131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767108.231134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767108.231135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767108.231139:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2996 00000100:00080000:1.0:1775767108.231141:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767108.231144:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767108.231154:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767108.231163:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742027776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767108.231171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767108.231174:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767108.231175:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767108.231180:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2996 00000100:00080000:1.0:1775767108.231182:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767108.231182:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742027648:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767108.231186:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767108.231194:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767108.231202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742027904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767108.231208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767108.231210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767108.231211:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767108.231214:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 2996 00000100:00080000:1.0:1775767108.231216:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767108.231218:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767108.231221:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767108.231230:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742028032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767108.231273:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767108.231324:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767108/real 1775767108] req@ffff90c203496680 x1862023742027648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767108.231346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742027648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767108.231358:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767108.231363:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767108.231367:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742027648:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767108.231396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027776:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767108.231419:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767108.231431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767108/real 1775767108] req@ffff90c203494a80 x1862023742027776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767108.231445:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742027776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767108.231454:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767108.231457:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767108.231461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767108.231467:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027904:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767108.231521:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767108.231533:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767108/real 1775767108] req@ffff90c203494700 x1862023742027904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767108.231546:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742027904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767108.231554:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767108.231558:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767108.231561:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742027904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767108.231568:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028032:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767108.231590:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767108.231601:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767108/real 1775767108] req@ffff90c203495500 x1862023742028032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767108.231613:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742028032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767108.231631:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767108.231635:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767108.231639:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028032:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767109.255050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767110.279116:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767111.303015:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767112.327125:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767113.350956:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767113.350964:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767113.350968:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767113.350978:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3001 00000100:00080000:1.0:1775767113.350989:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767113.350995:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767113.351013:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767113.351031:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742028160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767113.351056:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767113.351059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767113.351060:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767113.351063:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3001 00000100:00080000:1.0:1775767113.351065:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767113.351068:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767113.351076:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767113.351086:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742028288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767113.351094:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767113.351097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767113.351098:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767113.351100:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3001 00000100:00080000:1.0:1775767113.351103:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767113.351111:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767113.351114:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767113.351115:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742028160:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767113.351122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742028416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767113.351136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767113.351138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767113.351139:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767113.351142:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3001 00000100:00080000:1.0:1775767113.351145:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767113.351148:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767113.351151:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767113.351158:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742028544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767113.351227:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767113.351272:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767113/real 1775767113] req@ffff90c203495180 x1862023742028160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767113.351289:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742028160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767113.351299:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767113.351302:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767113.351305:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742028160:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767113.351334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767113.351369:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767113.351383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767113/real 1775767113] req@ffff90c203495500 x1862023742028288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767113.351393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742028288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767113.351399:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767113.351401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767113.351404:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767113.351409:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767113.351443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767113.351456:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767113/real 1775767113] req@ffff90c203494700 x1862023742028416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767113.351470:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742028416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767113.351479:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767113.351483:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767113.351486:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028416:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767113.351491:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028544:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767113.351509:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767113.351516:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767113/real 1775767113] req@ffff90c203494a80 x1862023742028544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767113.351524:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742028544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767113.351532:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767113.351535:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767113.351536:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028544:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767114.375117:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767115.399036:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767116.423076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767117.447077:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767118.471003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767118.471014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767118.471019:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767118.471028:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3006 00010000:00010000:0.0:1775767118.471043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767118.471046:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767118.471053:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767118.471087:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742028672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767118.471111:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767118.471114:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767118.471115:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767118.471118:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3006 00000100:00080000:1.0:1775767118.471121:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767118.471123:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767118.471131:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767118.471143:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742028800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767118.471151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767118.471155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767118.471158:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767118.471159:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742028672:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767118.471170:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3006 00000100:00080000:1.0:1775767118.471176:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767118.471179:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767118.471182:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767118.471191:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742028928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767118.471197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767118.471200:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767118.471201:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767118.471204:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3006 00000100:00080000:1.0:1775767118.471206:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767118.471209:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767118.471213:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767118.471220:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742029056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767118.471251:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767118.471292:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767118/real 1775767118] req@ffff90c203496680 x1862023742028672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767118.471318:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742028672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767118.471328:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767118.471332:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767118.471335:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742028672:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767118.471360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767118.471419:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767118.471433:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767118/real 1775767118] req@ffff90c203496a00 x1862023742028800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767118.471443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742028800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767118.471449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767118.471452:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767118.471454:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767118.471459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028928:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767118.471474:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767118.471481:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767118/real 1775767118] req@ffff90c203496300 x1862023742028928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767118.471489:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742028928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767118.471499:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767118.471502:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767118.471504:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742028928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767118.471509:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029056:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767118.471528:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767118.471549:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767118/real 1775767118] req@ffff90c203497b80 x1862023742029056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767118.471561:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742029056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767118.471573:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767118.471577:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767118.471581:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029056:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767119.495214:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767120.519036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767121.543148:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767122.567123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767123.590980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767123.590990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767123.590994:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767123.591005:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3012 00000100:00080000:1.0:1775767123.591016:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767123.591023:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767123.591048:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767123.591064:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742029184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767123.591085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767123.591090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767123.591092:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767123.591096:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3012 00000100:00080000:1.0:1775767123.591100:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767123.591115:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767123.591125:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767123.591135:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742029184:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767123.591137:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742029312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767123.591147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767123.591150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767123.591152:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767123.591156:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3012 00000100:00080000:1.0:1775767123.591158:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767123.591162:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767123.591166:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767123.591174:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742029440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767123.591182:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767123.591185:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767123.591186:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767123.591189:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3012 00000100:00080000:1.0:1775767123.591192:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767123.591192:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767123.591196:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767123.591200:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767123.591209:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742029568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767123.591230:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767123/real 1775767123] req@ffff90c203496d80 x1862023742029184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767123.591248:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742029184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767123.591286:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767123.591290:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767123.591294:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742029184:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767123.591351:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767123.591373:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767123.591383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767123/real 1775767123] req@ffff90c203495880 x1862023742029312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767123.591409:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742029312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767123.591415:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767123.591417:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767123.591419:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767123.591423:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767123.591435:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767123.591440:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767123/real 1775767123] req@ffff90c203497100 x1862023742029440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767123.591447:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742029440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767123.591452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767123.591453:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767123.591455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767123.591458:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029568:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767123.591468:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767123.591473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767123/real 1775767123] req@ffff90c203495c00 x1862023742029568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767123.591480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742029568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767123.591487:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767123.591488:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767123.591490:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029568:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767124.615036:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767125.639138:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767126.663022:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767127.687163:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767128.711037:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767128.711045:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767128.711053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767128.711059:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767128.711069:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3017 00000100:00080000:1.0:1775767128.711081:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767128.711088:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767128.711134:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742029696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767128.711161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767128.711164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767128.711166:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767128.711169:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3017 00000100:00080000:1.0:1775767128.711172:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767128.711174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767128.711184:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767128.711198:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742029824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767128.711208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767128.711211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767128.711212:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742029696:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767128.711213:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767128.711218:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3017 00000100:00080000:1.0:1775767128.711222:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767128.711226:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767128.711236:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767128.711249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742029952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767128.711256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767128.711258:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767128.711259:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767128.711278:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3017 00000100:00080000:1.0:1775767128.711287:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767128.711292:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767128.711297:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767128.711307:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742030080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767128.711318:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767128.711361:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767128/real 1775767128] req@ffff90c203494380 x1862023742029696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767128.711377:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742029696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767128.711386:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767128.711390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767128.711393:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742029696:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767128.711418:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767128.711443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767128.711457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767128/real 1775767128] req@ffff90c203494e00 x1862023742029824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767128.711471:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742029824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767128.711489:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767128.711494:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767128.711497:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767128.711504:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767128.711523:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767128.711545:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767128/real 1775767128] req@ffff90c203495f80 x1862023742029952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767128.711554:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742029952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767128.711560:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767128.711562:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767128.711564:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742029952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767128.711569:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030080:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767128.711582:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767128.711588:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767128/real 1775767128] req@ffff90c203494000 x1862023742030080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767153 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767128.711596:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742030080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767153 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767128.711603:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767128.711605:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767128.711608:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030080:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767129.735149:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767130.759016:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767131.783094:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767132.807015:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767133.831040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767133.831053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767133.831060:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:0.0:1775767133.831068:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767133.831075:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3022 00000100:00080000:1.0:1775767133.831091:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767133.831100:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767133.831157:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742030208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767133.831192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767133.831196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767133.831198:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767133.831202:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3022 00000100:00080000:1.0:1775767133.831206:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767133.831229:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767133.831241:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767133.831255:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742030208:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767133.831266:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742030336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767133.831288:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767133.831294:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767133.831297:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767133.831303:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3022 00000100:00080000:1.0:1775767133.831309:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767133.831316:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767133.831326:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767133.831331:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767133.831354:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742030464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767133.831372:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767133.831373:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767133/real 1775767133] req@ffff90c203497480 x1862023742030208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767133.831392:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767133.831399:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767133.831416:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3022 00000100:00100000:2.0:1775767133.831416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742030208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767133.831437:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767133.831441:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767133.831449:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775767133.831452:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767133.831465:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742030208:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775767133.831468:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767133.831489:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767133.831492:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742030592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767133.831513:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767133.831529:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767133/real 1775767133] req@ffff90c203494000 x1862023742030336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767133.831538:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742030336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767133.831546:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767133.831549:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767133.831553:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767133.831558:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030464:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767133.831569:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767133.831575:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767133/real 1775767133] req@ffff90c203495f80 x1862023742030464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767133.831583:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742030464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767133.831588:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767133.831590:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767133.831592:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767133.831600:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030592:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767133.831611:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767133.831617:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767133/real 1775767133] req@ffff90c203494e00 x1862023742030592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767133.831624:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742030592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767158 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767133.831639:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767133.831644:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767133.831646:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030592:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767134.855032:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767135.879049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767136.903138:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767137.927074:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767138.951058:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767138.951104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:3.0:1775767138.951112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767138.951118:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767138.951132:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3027 00000100:00080000:2.0:1775767138.951146:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767138.951154:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767138.951248:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023742030720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767138.951281:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767138.951284:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767138.951286:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767138.951290:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3027 00000100:00080000:2.0:1775767138.951293:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767138.951313:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767138.951324:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767138.951336:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023742030848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767138.951345:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775767138.951348:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742030720:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775767138.951350:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767138.951354:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767138.951362:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3027 00000100:00080000:2.0:1775767138.951364:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767138.951367:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767138.951370:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767138.951380:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023742030976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767138.951388:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767138.951390:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767138.951392:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767138.951396:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3027 00000100:00080000:2.0:1775767138.951399:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767138.951402:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767138.951407:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767138.951415:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023742031104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767138.951455:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767138.951520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767138/real 1775767138] req@ffff90c2034bd180 x1862023742030720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767138.951538:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023742030720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767138.951547:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767138.951551:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767138.951555:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742030720:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767138.951579:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767138.951596:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767138.951605:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767138/real 1775767138] req@ffff90c2034bdf80 x1862023742030848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767138.951614:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023742030848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767138.951620:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767138.951622:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767138.951625:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767138.951628:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767138.951641:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767138.951647:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767138/real 1775767138] req@ffff90c2034bf100 x1862023742030976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767138.951655:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023742030976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767138.951666:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767138.951669:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767138.951671:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742030976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767138.951675:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031104:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767138.951686:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767138.951691:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767138/real 1775767138] req@ffff90c2034bc700 x1862023742031104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767138.951698:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023742031104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767163 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767138.951706:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767138.951709:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767138.951711:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031104:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767139.975230:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767140.999132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767142.023223:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767143.047105:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767144.070996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767144.071006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767144.071012:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767144.071022:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3032 00000100:00080000:1.0:1775767144.071035:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767144.071042:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767144.071066:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767144.071090:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742031232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767144.071147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767144.071150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767144.071152:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767144.071155:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3032 00000100:00080000:1.0:1775767144.071158:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767144.071161:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767144.071171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767144.071183:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742031360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767144.071192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767144.071194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767144.071196:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767144.071201:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3032 00000100:00100000:0.0:1775767144.071203:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742031232:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767144.071204:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767144.071211:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767144.071214:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767144.071222:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742031488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767144.071228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767144.071230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767144.071232:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767144.071235:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3032 00000100:00080000:1.0:1775767144.071237:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767144.071239:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767144.071243:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767144.071249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742031616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767144.071277:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767144.071320:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767144/real 1775767144] req@ffff90c203494380 x1862023742031232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767144.071338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742031232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767144.071350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767144.071356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767144.071361:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742031232:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767144.071395:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767144.071432:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767144.071446:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767144/real 1775767144] req@ffff90c203495c00 x1862023742031360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767144.071459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742031360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767144.071467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767144.071470:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767144.071474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767144.071480:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031488:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767144.071501:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767144.071511:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767144/real 1775767144] req@ffff90c203497100 x1862023742031488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767144.071523:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742031488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767144.071533:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767144.071536:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767144.071540:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767144.071546:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031616:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767144.071563:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767144.071572:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767144/real 1775767144] req@ffff90c203495880 x1862023742031616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767144.071584:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742031616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767144.071595:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767144.071598:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767144.071602:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031616:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767145.095129:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767146.119172:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767147.143170:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767148.167188:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767149.191126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767149.191144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767149.191152:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767149.191168:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3037 00000100:00080000:1.0:1775767149.191187:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775767149.191196:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767149.191197:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767149.191273:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742031744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767149.191310:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767149.191315:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767149.191317:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767149.191323:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3037 00000100:00080000:1.0:1775767149.191330:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767149.191335:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767149.191349:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767149.191351:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742031744:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767149.191373:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742031872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767149.191389:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767149.191395:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767149.191397:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767149.191402:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3037 00000100:00080000:1.0:1775767149.191420:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767149.191425:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767149.191431:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767149.191438:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767149.191446:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742032000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767149.191456:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767149.191459:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767149.191461:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767149.191466:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3037 00000100:00080000:1.0:1775767149.191469:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767149.191471:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767149.191476:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767149.191487:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742032128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767149.191495:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767149/real 1775767149] req@ffff90c203496d80 x1862023742031744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767149.191519:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742031744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767149.191530:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767149.191535:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767149.191538:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742031744:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767149.191564:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031872:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767149.191583:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767149.191592:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767149/real 1775767149] req@ffff90c203497b80 x1862023742031872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767149.191608:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742031872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767149.191615:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767149.191618:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767149.191621:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742031872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767149.191625:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767149.191637:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767149.191643:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767149/real 1775767149] req@ffff90c203496300 x1862023742032000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767149.191652:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742032000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767149.191658:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767149.191660:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767149.191663:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767149.191667:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032128:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767149.191679:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767149.191685:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767149/real 1775767149] req@ffff90c203496a00 x1862023742032128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767149.191693:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742032128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767149.191714:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767149.191719:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767149.191723:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032128:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767150.215105:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767151.239131:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767152.263063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767153.287098:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767154.311032:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767154.311047:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767154.311054:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767154.311068:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3042 00000100:00080000:1.0:1775767154.311082:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767154.311090:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767154.311091:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767154.311140:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742032256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767154.311175:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767154.311178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767154.311180:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767154.311191:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3042 00000100:00080000:1.0:1775767154.311195:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767154.311201:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767154.311212:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767154.311225:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742032384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767154.311234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767154.311237:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767154.311239:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767154.311243:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742032256:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767154.311245:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3042 00000100:00080000:1.0:1775767154.311250:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767154.311256:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767154.311259:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767154.311268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742032512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767154.311278:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767154.311281:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767154.311282:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767154.311287:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3042 00000100:00080000:1.0:1775767154.311290:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767154.311293:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767154.311297:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767154.311306:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742032640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767154.311359:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767154.311410:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767154/real 1775767154] req@ffff90c203496680 x1862023742032256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767154.311436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742032256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767154.311449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767154.311455:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767154.311460:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742032256:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767154.311500:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767154.311525:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767154.311538:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767154/real 1775767154] req@ffff90c203494a80 x1862023742032384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767154.311551:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742032384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767154.311561:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767154.311565:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767154.311568:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767154.311576:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767154.311597:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767154.311608:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767154/real 1775767154] req@ffff90c203494700 x1862023742032512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767154.311621:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742032512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767154.311629:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767154.311633:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767154.311647:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767154.311653:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032640:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767154.311672:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767154.311683:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767154/real 1775767154] req@ffff90c203495500 x1862023742032640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767154.311696:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742032640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767154.311707:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767154.311711:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767154.311715:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032640:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767155.335095:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767156.359147:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767157.383055:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767158.407056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767159.431018:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767159.431030:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767159.431036:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775767159.431041:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767159.431049:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3047 00000100:00080000:0.0:1775767159.431063:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767159.431072:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767159.431123:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742032768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767159.431161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767159.431166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767159.431169:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767159.431178:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3047 00000100:00080000:0.0:1775767159.431184:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767159.431189:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767159.431204:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767159.431204:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742032768:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767159.431233:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742032896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767159.431249:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767159.431253:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767159.431255:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767159.431259:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3047 00000100:00080000:0.0:1775767159.431265:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767159.431271:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767159.431277:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767159.431288:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742033024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767159.431303:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767159.431307:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767159.431309:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767159.431313:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3047 00000100:00080000:0.0:1775767159.431318:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767159.431323:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032896:192.168.202.155@tcp:38:kworker.0 00000100:00080000:0.0:1775767159.431326:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767159.431338:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767159.431354:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742033152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767159.431399:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767159.431438:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033152:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767160.455098:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767161.479115:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767162.503139:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775767162.503224:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775767162.503274:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775767162.503291:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767162.503333:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767162.503343:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767162.503356:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767162.503397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767159/real 1775767162] req@ffff90c203638e00 x1862023742033152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767162.503437:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742033152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767162.503458:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767162.503465:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767162.503471:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033152:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767162.503482:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767159/real 1775767162] req@ffff90c20363bb80 x1862023742032896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767162.503498:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742032896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767162.503509:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767162.503514:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767162.503517:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742032896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767162.503526:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767159/real 1775767162] req@ffff90c20363b100 x1862023742033024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767162.503539:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742033024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767162.503549:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767162.503553:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767162.503556:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767162.503564:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767159/real 1775767162] req@ffff90c20363a680 x1862023742032768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767162.503578:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742032768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767162.503587:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767162.503591:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767162.503594:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742032768:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767163.527092:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767164.551017:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767164.551027:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767164.551031:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767164.551041:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3053 00000100:00080000:0.0:1775767164.551052:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767164.551061:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767164.551097:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742033280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767164.551097:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767164.551132:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767164.551136:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767164.551138:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767164.551142:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3053 00000100:00080000:0.0:1775767164.551146:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767164.551151:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767164.551160:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767164.551172:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742033408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767164.551180:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767164.551183:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767164.551184:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767164.551187:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3053 00000100:00080000:0.0:1775767164.551190:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767164.551195:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767164.551199:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742033280:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775767164.551204:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767164.551217:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742033536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767164.551223:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767164.551226:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767164.551227:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767164.551230:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3053 00000100:00080000:0.0:1775767164.551233:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767164.551236:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767164.551241:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767164.551250:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742033664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767164.551321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767164.551383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767164/real 1775767164] req@ffff90c203639500 x1862023742033280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767164.551411:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742033280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767164.551425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767164.551431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767164.551436:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742033280:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767164.551475:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767164.551505:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767164.551520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767164/real 1775767164] req@ffff90c203638000 x1862023742033408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767164.551533:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742033408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767164.551543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767164.551546:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767164.551550:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767164.551557:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033536:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767164.551579:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767164.551591:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767164/real 1775767164] req@ffff90c203639180 x1862023742033536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767164.551604:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742033536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767164.551618:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767164.551622:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767164.551625:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767164.551632:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033664:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767164.551654:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767164.551665:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767164/real 1775767164] req@ffff90c20363aa00 x1862023742033664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767164.551676:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742033664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767164.551688:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767164.551691:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767164.551694:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033664:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767165.575064:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767166.599112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767167.623055:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767168.647116:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767169.670999:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767169.671010:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:3.0:1775767169.671012:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767169.671027:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767169.671040:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3058 00000100:00080000:2.0:1775767169.671054:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767169.671063:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767169.671135:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bed80 x1862023742033792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767169.671162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767169.671166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767169.671168:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767169.671171:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3058 00000100:00080000:2.0:1775767169.671175:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767169.671179:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767169.671200:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767169.671207:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742033792:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767169.671216:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be680 x1862023742033920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767169.671229:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767169.671234:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767169.671236:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767169.671241:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3058 00000100:00080000:2.0:1775767169.671246:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767169.671249:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767169.671254:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767169.671266:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023742034048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767169.671273:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767169.671276:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767169.671277:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775767169.671282:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775767169.671283:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3058 00000100:00080000:2.0:1775767169.671289:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767169.671292:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767169.671297:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767169.671305:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023742034176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767169.671335:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767169/real 1775767169] req@ffff90c2034bed80 x1862023742033792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767169.671358:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bed80 x1862023742033792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767169.671367:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767169.671371:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767169.671374:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742033792:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767169.671398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767169.671415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767169.671424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767169/real 1775767169] req@ffff90c2034be680 x1862023742033920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767169.671433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be680 x1862023742033920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767169.671439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767169.671441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767169.671444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742033920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767169.671448:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034048:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767169.671459:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767169.671465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767169/real 1775767169] req@ffff90c2034bc700 x1862023742034048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767169.671472:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023742034048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767169.671477:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767169.671479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767169.671481:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767169.671484:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034176:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767169.671494:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767169.671499:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767169/real 1775767169] req@ffff90c2034bf100 x1862023742034176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767194 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767169.671505:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023742034176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767194 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767169.671513:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767169.671515:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767169.671516:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034176:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767170.695057:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767171.719057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767172.743063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767173.767005:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767174.791002:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767174.791012:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767174.791018:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767174.791030:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3063 00000100:00080000:1.0:1775767174.791043:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767174.791053:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767174.791078:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767174.791091:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742034304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767174.791116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767174.791120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767174.791122:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767174.791125:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3063 00000100:00080000:1.0:1775767174.791127:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767174.791130:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767174.791138:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767174.791149:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742034432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767174.791158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767174.791160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767174.791163:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767174.791164:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742034304:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767174.791168:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3063 00000100:00080000:1.0:1775767174.791173:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767174.791177:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767174.791180:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767174.791188:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742034560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767174.791195:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767174.791198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767174.791199:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767174.791202:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3063 00000100:00080000:1.0:1775767174.791214:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767174.791216:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767174.791221:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775767174.791227:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767174.791230:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742034688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767174.791270:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767174/real 1775767174] req@ffff90c203495180 x1862023742034304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767174.791291:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742034304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767174.791302:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767174.791307:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767174.791311:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742034304:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767174.791331:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034432:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767174.791352:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767174.791364:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767174/real 1775767174] req@ffff90c203495500 x1862023742034432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767174.791378:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742034432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767174.791387:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767174.791391:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767174.791394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767174.791401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034560:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767174.791417:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767174.791424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767174/real 1775767174] req@ffff90c203494700 x1862023742034560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767174.791432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742034560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767174.791442:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767174.791444:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767174.791446:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767174.791450:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034688:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767174.791459:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767174.791464:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767174/real 1775767174] req@ffff90c203494a80 x1862023742034688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767199 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767174.791471:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742034688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767199 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767174.791477:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767174.791479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767174.791481:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034688:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767175.815075:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767176.839047:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767177.863061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767178.887036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767179.911026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767179.911039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767179.911045:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767179.911058:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3068 00010000:00010000:3.0:1775767179.911058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767179.911073:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767179.911083:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767179.911128:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742034816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767179.911152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767179.911155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767179.911157:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767179.911160:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3068 00000100:00080000:1.0:1775767179.911164:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767179.911168:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767179.911177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767179.911186:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742034944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767179.911198:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742034816:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767179.911200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767179.911217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767179.911219:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767179.911223:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3068 00000100:00080000:1.0:1775767179.911226:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767179.911231:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767179.911234:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767179.911243:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742035072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767179.911250:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767179.911253:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767179.911254:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767179.911260:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3068 00000100:00080000:1.0:1775767179.911268:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767179.911272:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767179.911275:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775767179.911282:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767179.911286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742035200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767179.911329:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767179/real 1775767179] req@ffff90c203496680 x1862023742034816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767179.911343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742034816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767179.911352:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767179.911355:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767179.911358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742034816:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767179.911377:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767179.911392:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767179.911399:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767179/real 1775767179] req@ffff90c203496a00 x1862023742034944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767179.911407:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742034944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767179.911412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767179.911415:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767179.911417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742034944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767179.911421:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767179.911430:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767179.911435:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767179/real 1775767179] req@ffff90c203496300 x1862023742035072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767179.911443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742035072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767179.911447:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767179.911449:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767179.911451:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767179.911455:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035200:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767179.911464:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767179.911469:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767179/real 1775767179] req@ffff90c203497b80 x1862023742035200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767179.911476:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742035200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767204 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767179.911483:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767179.911485:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767179.911486:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035200:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767180.935041:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767181.959074:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767182.983028:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767184.007076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767185.030980:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767185.030994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767185.031001:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767185.031015:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3073 00000100:00080000:2.0:1775767185.031030:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767185.031039:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767185.031097:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023742035328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767185.031130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767185.031135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767185.031137:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767185.031142:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3073 00000100:00080000:2.0:1775767185.031146:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767185.031151:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767185.031164:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767185.031178:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742035328:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767185.031181:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023742035456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767185.031193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767185.031196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767185.031197:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767185.031201:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3073 00000100:00080000:2.0:1775767185.031205:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767185.031208:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767185.031211:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767185.031221:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023742035584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767185.031228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767185.031230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767185.031231:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767185.031234:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3073 00000100:00080000:2.0:1775767185.031237:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767185.031239:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767185.031242:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767185.031249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023742035712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767185.031260:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767185.031306:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767185/real 1775767185] req@ffff90c2034bdf80 x1862023742035328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767185.031324:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023742035328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767185.031334:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00010000:00010000:3.0:1775767185.031336:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767185.031338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767185.031343:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742035328:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767185.031369:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767185.031385:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767185.031392:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767185/real 1775767185] req@ffff90c2034bd180 x1862023742035456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767185.031400:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023742035456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767185.031411:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767185.031413:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767185.031417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767185.031423:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035584:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767185.031444:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767185.031455:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767185/real 1775767185] req@ffff90c2034bc380 x1862023742035584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767185.031468:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023742035584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767185.031477:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767185.031480:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767185.031484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767185.031490:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035712:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767185.031512:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767185.031523:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767185/real 1775767185] req@ffff90c2034bce00 x1862023742035712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767185.031531:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023742035712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767185.031539:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767185.031541:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767185.031544:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035712:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767186.055082:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767187.079064:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767188.103136:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767189.127071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767190.150994:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767190.151003:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767190.151007:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767190.151015:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3078 00000100:00080000:2.0:1775767190.151025:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767190.151031:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767190.151071:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023742035840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767190.151095:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767190.151100:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767190.151101:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767190.151106:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3078 00000100:00080000:2.0:1775767190.151110:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767190.151113:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767190.151123:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767190.151139:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023742035968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767190.151151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767190.151155:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767190.151157:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767190.151161:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3078 00000100:00080000:2.0:1775767190.151165:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767190.151173:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767190.151179:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742035840:192.168.202.155@tcp:250:kworker.0 00000008:00100000:2.0:1775767190.151181:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767190.151199:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742036096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767190.151214:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767190.151219:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767190.151221:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767190.151226:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3078 00000100:00080000:2.0:1775767190.151232:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767190.151237:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767190.151243:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767190.151257:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742036224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767190.151274:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767190.151327:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767190/real 1775767190] req@ffff90c2034bf480 x1862023742035840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767190.151346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023742035840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767190.151357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767190.151362:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767190.151366:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742035840:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767190.151369:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767190.151374:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767190.151391:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767190.151400:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767190/real 1775767190] req@ffff90c2034be300 x1862023742035968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767190.151408:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023742035968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767190.151413:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767190.151416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767190.151418:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742035968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767190.151442:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767190.151454:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767190.151460:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767190/real 1775767190] req@ffff90c2034bca80 x1862023742036096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767190.151468:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742036096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767190.151473:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767190.151475:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767190.151477:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767190.151481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036224:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767190.151492:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767190.151497:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767190/real 1775767190] req@ffff90c2034bea00 x1862023742036224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767190.151505:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742036224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767190.151512:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767190.151514:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767190.151516:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036224:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767191.175098:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767192.199115:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767193.223087:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767194.247089:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767195.271079:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767195.271091:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767195.271098:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767195.271111:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3083 00000100:00080000:2.0:1775767195.271125:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775767195.271130:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767195.271132:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767195.271185:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bfb80 x1862023742036352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767195.271210:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767195.271214:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767195.271215:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767195.271218:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3083 00000100:00080000:2.0:1775767195.271221:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767195.271223:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767195.271232:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767195.271240:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd500 x1862023742036480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767195.271247:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767195.271250:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767195.271251:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767195.271253:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3083 00000100:00080000:2.0:1775767195.271256:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767195.271259:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767195.271262:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767195.271270:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742036608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767195.271277:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767195.271280:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767195.271281:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767195.271285:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3083 00000100:00080000:2.0:1775767195.271290:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767195.271295:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767195.271301:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767195.271316:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742036736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767195.271341:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742036352:192.168.202.155@tcp:250:kworker.0 00000800:00000100:0.0:1775767195.271429:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767195.271475:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767195/real 1775767195] req@ffff90c2034bfb80 x1862023742036352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767195.271502:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bfb80 x1862023742036352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767195.271518:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767195.271525:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767195.271531:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742036352:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767195.271542:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767195.271592:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767195.271607:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767195/real 1775767195] req@ffff90c2034bd500 x1862023742036480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767195.271619:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd500 x1862023742036480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767195.271627:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767195.271630:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767195.271633:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767195.271638:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767195.271656:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767195.271663:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767195/real 1775767195] req@ffff90c2034bea00 x1862023742036608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767195.271672:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742036608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767195.271679:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767195.271682:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767195.271684:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767195.271712:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036736:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767195.271727:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767195.271734:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767195/real 1775767195] req@ffff90c2034bca80 x1862023742036736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767195.271744:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742036736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767195.271754:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767195.271756:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767195.271759:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036736:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767196.295079:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767197.319080:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767198.343067:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767199.367079:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767200.391003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767200.391016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767200.391022:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767200.391036:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3088 00000100:00080000:1.0:1775767200.391050:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767200.391059:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767200.391086:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767200.391109:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742036864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767200.391135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767200.391138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767200.391140:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767200.391143:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3088 00000100:00080000:1.0:1775767200.391146:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767200.391149:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767200.391160:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767200.391170:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742036992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767200.391180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767200.391183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1775767200.391184:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742036864:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767200.391184:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767200.391192:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3088 00000100:00080000:1.0:1775767200.391195:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767200.391197:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767200.391201:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767200.391208:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742037120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767200.391214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767200.391217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767200.391218:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767200.391222:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3088 00000100:00080000:1.0:1775767200.391225:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767200.391227:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767200.391231:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767200.391240:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742037248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767200.391264:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767200.391308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767200/real 1775767200] req@ffff90c203496d80 x1862023742036864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767200.391323:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742036864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767200.391331:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767200.391335:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767200.391338:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742036864:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767200.391359:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036992:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767200.391374:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767200.391381:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767200/real 1775767200] req@ffff90c203495880 x1862023742036992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767200.391390:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742036992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767200.391396:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767200.391398:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767200.391400:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742036992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767200.391403:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767200.391413:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767200.391418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767200/real 1775767200] req@ffff90c203497100 x1862023742037120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767200.391425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742037120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767200.391430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767200.391432:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767200.391433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767200.391437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037248:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767200.391447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767200.391451:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767200/real 1775767200] req@ffff90c203495c00 x1862023742037248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767200.391457:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742037248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767200.391469:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767200.391471:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767200.391473:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037248:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767201.415021:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767202.439048:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767203.463105:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767204.487060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767205.511044:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767205.511052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767205.511057:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767205.511069:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3093 00010000:00010000:1.0:1775767205.511070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767205.511083:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767205.511090:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767205.511125:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742037376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767205.511156:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767205.511161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767205.511163:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767205.511167:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3093 00000100:00080000:3.0:1775767205.511171:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767205.511175:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767205.511185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767205.511200:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742037504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767205.511210:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742037376:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767205.511215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767205.511220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767205.511222:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767205.511225:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3093 00000100:00080000:3.0:1775767205.511229:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767205.511232:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767205.511237:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767205.511246:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742037632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767205.511253:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767205.511255:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767205.511257:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767205.511259:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3093 00000100:00080000:3.0:1775767205.511261:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767205.511263:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767205.511266:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767205.511275:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742037760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767205.511280:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767205.511319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767205/real 1775767205] req@ffff90c218da3480 x1862023742037376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767205.511333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742037376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767205.511341:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767205.511344:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767205.511347:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742037376:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767205.511372:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767205.511397:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767205.511410:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767205/real 1775767205] req@ffff90c218da0700 x1862023742037504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767205.511420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742037504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767205.511426:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767205.511428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767205.511430:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767205.511435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767205.511449:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767205.511455:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767205/real 1775767205] req@ffff90c218da0380 x1862023742037632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767205.511463:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742037632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767205.511467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767205.511469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767205.511471:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767205.511501:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037760:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767205.511517:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767205.511524:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767205/real 1775767205] req@ffff90c218da0a80 x1862023742037760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767205.511532:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742037760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767205.511542:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767205.511545:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767205.511548:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742037760:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767206.535094:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767207.559059:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767208.583042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767209.607215:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767210.631020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767210.631032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767210.631037:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767210.631048:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3099 00000100:00080000:3.0:1775767210.631060:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767210.631067:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775767210.631099:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767210.631123:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742037888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767210.631153:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767210.631156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767210.631158:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767210.631161:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3099 00000100:00080000:3.0:1775767210.631165:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767210.631168:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767210.631180:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767210.631184:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742037888:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767210.631195:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742038016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767210.631204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767210.631207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767210.631208:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767210.631211:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3099 00000100:00080000:3.0:1775767210.631213:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767210.631215:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767210.631219:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767210.631226:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742038144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767210.631233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767210.631236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767210.631237:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767210.631240:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3099 00000100:00080000:3.0:1775767210.631243:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767210.631245:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767210.631249:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767210.631257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742038272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767210.631262:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767210.631312:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767210/real 1775767210] req@ffff90c218da3100 x1862023742037888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767210.631336:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742037888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767210.631349:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767210.631354:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767210.631358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742037888:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767210.631383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767210.631402:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767210.631411:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767210/real 1775767210] req@ffff90c218da0000 x1862023742038016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767210.631420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742038016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767210.631450:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767210.631453:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767210.631456:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767210.631460:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767210.631476:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767210.631484:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767210/real 1775767210] req@ffff90c218da1f80 x1862023742038144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767210.631492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742038144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767210.631502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767210.631504:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767210.631507:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767210.631510:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038272:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767210.631520:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767210.631525:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767210/real 1775767210] req@ffff90c218da0a80 x1862023742038272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767235 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767210.631532:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742038272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767235 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767210.631540:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767210.631541:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767210.631543:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038272:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767211.655137:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767212.679117:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767213.703097:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767214.727105:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767215.751062:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775767215.751064:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767215.751075:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767215.751081:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767215.751095:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3104 00000100:00080000:2.0:1775767215.751107:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767215.751113:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767215.751158:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023742038400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767215.751185:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767215.751189:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767215.751191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767215.751194:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3104 00000100:00080000:2.0:1775767215.751196:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767215.751199:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767215.751208:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767215.751217:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023742038528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767215.751225:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767215.751228:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767215.751229:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767215.751232:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3104 00000100:00080000:2.0:1775767215.751234:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767215.751236:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767215.751239:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767215.751245:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023742038656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767215.751256:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742038400:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775767215.751257:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767215.751266:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767215.751268:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767215.751284:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3104 00000100:00080000:2.0:1775767215.751288:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767215.751293:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767215.751301:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767215.751315:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023742038784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767215.751346:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767215.751411:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767215/real 1775767215] req@ffff90c2034be300 x1862023742038400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767215.751432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023742038400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767215.751443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767215.751448:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767215.751453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742038400:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767215.751462:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038528:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767215.751486:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767215.751499:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767215/real 1775767215] req@ffff90c2034bf480 x1862023742038528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767215.751510:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023742038528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767215.751519:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767215.751523:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767215.751526:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767215.751553:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767215.751573:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767215.751582:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767215/real 1775767215] req@ffff90c2034bce00 x1862023742038656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767215.751594:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023742038656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767215.751601:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767215.751605:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767215.751608:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767215.751613:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038784:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767215.751632:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767215.751641:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767215/real 1775767215] req@ffff90c2034bc380 x1862023742038784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767240 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767215.751653:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023742038784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767240 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767215.751665:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767215.751668:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767215.751671:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742038784:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767216.775109:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767217.799142:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767218.823104:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767219.847040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767220.871046:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767220.871069:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767220.871075:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767220.871086:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3109 00000100:00080000:2.0:1775767220.871099:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767220.871106:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767220.871156:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023742038912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767220.871188:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767220.871191:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767220.871193:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767220.871196:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3109 00000100:00080000:2.0:1775767220.871199:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767220.871201:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767220.871211:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767220.871220:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023742039040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767220.871228:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767220.871231:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767220.871232:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767220.871235:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3109 00000100:00080000:2.0:1775767220.871238:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767220.871241:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767220.871245:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767220.871251:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023742039168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767220.871258:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767220.871261:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767220.871262:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767220.871266:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3109 00000100:00080000:2.0:1775767220.871270:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767220.871273:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767220.871276:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767220.871283:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023742039296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767220.871324:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742038912:192.168.202.155@tcp:250:kworker.0 00000800:00000100:0.0:1775767220.871435:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767220.871474:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767220/real 1775767220] req@ffff90c2034bd180 x1862023742038912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767220.871494:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023742038912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767220.871507:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767220.871512:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767220.871520:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742038912:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775767220.871522:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767220.871547:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767220.871586:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767220.871599:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767220/real 1775767220] req@ffff90c2034bdf80 x1862023742039040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767220.871623:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023742039040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767220.871633:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767220.871638:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767220.871641:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767220.871649:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767220.871684:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767220.871696:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767220/real 1775767220] req@ffff90c2034bf100 x1862023742039168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767220.871711:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023742039168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767220.871720:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767220.871724:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767220.871728:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767220.871762:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039296:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767220.871795:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767220.871802:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767220/real 1775767220] req@ffff90c2034bc700 x1862023742039296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767220.871810:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023742039296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767245 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767220.871821:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767220.871824:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767220.871826:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039296:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767221.895083:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767222.919062:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767223.943012:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767224.967083:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767225.990965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767225.990979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767225.990985:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767225.990999:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3114 00000100:00080000:1.0:1775767225.991013:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767225.991022:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775767225.991045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767225.991087:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742039424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767225.991118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767225.991123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767225.991125:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767225.991131:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3114 00000100:00080000:1.0:1775767225.991136:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767225.991140:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767225.991148:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742039424:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775767225.991153:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767225.991171:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742039552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767225.991183:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767225.991188:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767225.991190:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767225.991194:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3114 00000100:00080000:1.0:1775767225.991209:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767225.991213:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767225.991219:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767225.991230:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742039680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767225.991240:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767225.991243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767225.991245:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767225.991249:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3114 00000100:00080000:1.0:1775767225.991253:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767225.991257:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767225.991262:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767225.991273:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742039808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767225.991281:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767225.991332:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767225.991355:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039808:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767227.015069:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767228.039073:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767229.063093:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775767229.063172:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775767229.063220:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767229.063247:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767229.063315:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767229.063326:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767229.063338:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775767229.063370:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767225/real 1775767229] req@ffff90c203494000 x1862023742039808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767229.063406:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742039808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767229.063424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767229.063430:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767229.063435:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039808:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775767229.063445:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767225/real 1775767229] req@ffff90c203494e00 x1862023742039552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767229.063459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742039552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767229.063468:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767229.063473:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767229.063476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767229.063484:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767225/real 1775767229] req@ffff90c203495f80 x1862023742039680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767229.063496:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742039680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767229.063504:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767229.063509:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767229.063512:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742039680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767229.063520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767225/real 1775767229] req@ffff90c203494380 x1862023742039424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767229.063533:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742039424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767250 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767229.063541:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767229.063545:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767229.063548:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742039424:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767230.087075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767231.111035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767231.111049:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767231.111055:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767231.111068:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3119 00000100:00080000:2.0:1775767231.111083:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767231.111092:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767231.111142:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be680 x1862023742039936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767231.111176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767231.111181:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767231.111184:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767231.111189:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3119 00000100:00080000:2.0:1775767231.111194:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767231.111200:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767231.111213:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742039936:192.168.202.155@tcp:250:kworker.0 00000008:00100000:2.0:1775767231.111214:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767231.111250:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bed80 x1862023742040064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767231.111268:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767231.111272:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767231.111275:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767231.111280:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3119 00000100:00080000:2.0:1775767231.111285:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767231.111290:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767231.111296:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767231.111307:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767231.111316:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023742040192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767231.111334:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767231.111338:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767231.111341:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767231.111346:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3119 00000100:00100000:0.0:1775767231.111350:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767231/real 1775767231] req@ffff90c2034be680 x1862023742039936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767231.111353:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767231.111362:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767231.111373:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767231.111385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be680 x1862023742039936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767231.111390:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023742040320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767231.111411:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767231.111419:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767231.111426:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742039936:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767231.111459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767231.111485:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767231.111505:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767231/real 1775767231] req@ffff90c2034bed80 x1862023742040064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767231.111518:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bed80 x1862023742040064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767231.111527:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767231.111530:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767231.111534:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767231.111540:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767231.111563:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00010000:00010000:3.0:1775767231.111573:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767231.111577:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767231/real 1775767231] req@ffff90c2034bc700 x1862023742040192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767231.111592:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023742040192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767231.111607:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767231.111610:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767231.111613:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767231.111619:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040320:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767231.111641:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767231.111652:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767231/real 1775767231] req@ffff90c2034bf100 x1862023742040320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767231.111664:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023742040320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767231.111676:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767231.111679:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767231.111682:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040320:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767232.135089:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767233.159118:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767234.183137:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767235.207039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767236.230965:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767236.230979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767236.230985:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767236.231000:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3124 00010000:00010000:3.0:1775767236.231004:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767236.231015:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767236.231026:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767236.231077:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742040448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767236.231106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767236.231109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767236.231110:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767236.231114:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3124 00000100:00080000:1.0:1775767236.231117:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767236.231120:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767236.231130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767236.231147:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742040576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767236.231166:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742040448:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767236.231173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767236.231178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767236.231180:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767236.231186:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3124 00000100:00080000:1.0:1775767236.231224:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767236.231230:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767236.231236:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767236.231250:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742040704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767236.231269:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767236.231271:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767236.231275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767236.231278:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767236.231285:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3124 00000100:00080000:1.0:1775767236.231290:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767236.231294:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767236.231300:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767236.231315:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742040832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767236.231319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767236/real 1775767236] req@ffff90c203497480 x1862023742040448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767236.231346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742040448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767236.231358:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767236.231363:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767236.231368:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742040448:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767236.231395:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767236.231418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767236.231430:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767236/real 1775767236] req@ffff90c203494380 x1862023742040576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767236.231443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742040576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767236.231453:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767236.231456:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767236.231459:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767236.231464:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767236.231478:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767236.231485:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767236/real 1775767236] req@ffff90c203495f80 x1862023742040704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767236.231493:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742040704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767236.231499:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767236.231501:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767236.231503:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767236.231506:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040832:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767236.231518:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767236.231523:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767236/real 1775767236] req@ffff90c203494e00 x1862023742040832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767236.231530:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742040832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767236.231538:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767236.231539:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767236.231541:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742040832:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767237.255083:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767238.279087:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767239.303062:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767240.327090:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767241.350968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767241.350978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767241.350983:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767241.350993:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3129 00000100:00080000:2.0:1775767241.351005:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767241.351012:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767241.351056:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023742040960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767241.351081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767241.351084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767241.351086:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767241.351089:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3129 00000100:00080000:2.0:1775767241.351091:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767241.351093:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767241.351102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767241.351110:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023742041088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767241.351117:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767241.351120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767241.351122:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767241.351124:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3129 00000100:00080000:2.0:1775767241.351127:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767241.351130:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767241.351133:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767241.351162:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023742041216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767241.351163:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742040960:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775767241.351178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767241.351180:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767241.351181:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767241.351184:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3129 00000100:00080000:2.0:1775767241.351187:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767241.351189:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767241.351193:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767241.351202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023742041344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767241.351246:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767241.351301:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767241/real 1775767241] req@ffff90c2034bdf80 x1862023742040960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1775767241.351301:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767241.351326:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023742040960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767241.351339:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767241.351343:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767241.351348:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742040960:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767241.351356:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041088:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767241.351378:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767241.351389:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767241/real 1775767241] req@ffff90c2034bd180 x1862023742041088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767241.351410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023742041088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767241.351418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767241.351422:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767241.351426:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767241.351452:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041216:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767241.351469:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767241.351479:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767241/real 1775767241] req@ffff90c2034bc380 x1862023742041216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767241.351492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023742041216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767241.351500:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767241.351504:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767241.351507:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767241.351514:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041344:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767241.351529:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767241.351537:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767241/real 1775767241] req@ffff90c2034bce00 x1862023742041344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767241.351549:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023742041344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767241.351567:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767241.351570:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767241.351574:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041344:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767242.375165:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767243.399100:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767244.423037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767245.447080:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767246.471016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767246.471025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767246.471029:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767246.471037:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3134 00000100:00080000:1.0:1775767246.471048:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767246.471054:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767246.471078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767246.471089:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742041472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767246.471129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767246.471132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767246.471133:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767246.471136:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3134 00000100:00080000:1.0:1775767246.471139:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767246.471142:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767246.471151:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767246.471163:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742041600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767246.471171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767246.471174:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767246.471175:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767246.471180:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3134 00000100:00080000:1.0:1775767246.471186:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767246.471188:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742041472:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767246.471192:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767246.471205:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767246.471213:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742041728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767246.471219:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767246.471221:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767246.471222:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767246.471226:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3134 00000100:00080000:1.0:1775767246.471228:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767246.471231:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767246.471236:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767246.471243:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742041856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767246.471293:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767246.471346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767246/real 1775767246] req@ffff90c203494000 x1862023742041472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767246.471368:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742041472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767246.471380:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767246.471385:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767246.471390:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742041472:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767246.471431:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767246.471457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767246.471471:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767246/real 1775767246] req@ffff90c203495c00 x1862023742041600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767246.471484:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742041600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767246.471493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767246.471497:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767246.471500:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767246.471508:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767246.471530:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767246.471540:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767246/real 1775767246] req@ffff90c203497100 x1862023742041728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767246.471553:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742041728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767246.471561:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767246.471565:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767246.471568:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767246.471574:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041856:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767246.471594:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767246.471604:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767246/real 1775767246] req@ffff90c203495880 x1862023742041856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767246.471616:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742041856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767246.471627:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767246.471630:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767246.471634:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742041856:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767247.495143:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767248.519114:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767249.543246:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767250.567109:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767251.591085:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767251.591096:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767251.591103:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767251.591115:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3140 00000100:00080000:0.0:1775767251.591128:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767251.591139:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767251.591185:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767251.591192:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742041984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767251.591231:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767251.591234:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767251.591235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767251.591238:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3140 00000100:00080000:0.0:1775767251.591240:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767251.591243:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767251.591252:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767251.591266:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742042112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767251.591270:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742041984:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767251.591281:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767251.591283:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767251.591284:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767251.591287:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3140 00000100:00080000:0.0:1775767251.591289:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767251.591291:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767251.591294:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767251.591303:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742042240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767251.591314:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767251.591317:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767251.591319:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767251.591322:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3140 00000100:00080000:0.0:1775767251.591324:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767251.591326:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767251.591329:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767251.591336:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742042368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767251.591353:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767251.591395:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767251/real 1775767251] req@ffff90c203638700 x1862023742041984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767251.591412:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742041984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767251.591421:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767251.591424:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767251.591428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742041984:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767251.591451:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767251.591465:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767251.591483:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767251/real 1775767251] req@ffff90c20363aa00 x1862023742042112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767251.591491:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742042112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767251.591496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767251.591499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767251.591501:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767251.591505:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042240:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767251.591515:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767251.591520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767251/real 1775767251] req@ffff90c203639180 x1862023742042240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767251.591527:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742042240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767251.591532:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767251.591533:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767251.591535:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767251.591538:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042368:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767251.591548:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767251.591553:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767251/real 1775767251] req@ffff90c203638000 x1862023742042368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767251.591561:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742042368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767251.591568:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767251.591570:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767251.591572:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042368:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767252.615232:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767253.639146:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767254.663105:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767255.687117:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767256.710993:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767256.711016:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767256.711021:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767256.711033:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3145 00000100:00080000:2.0:1775767256.711045:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767256.711052:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767256.711098:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023742042496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767256.711135:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767256.711139:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767256.711140:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767256.711143:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3145 00000100:00080000:2.0:1775767256.711146:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767256.711148:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767256.711157:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767256.711167:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023742042624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767256.711179:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742042496:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775767256.711183:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767256.711187:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767256.711188:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767256.711192:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3145 00000100:00080000:2.0:1775767256.711194:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767256.711196:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767256.711199:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767256.711206:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742042752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767256.711212:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767256.711215:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767256.711221:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767256.711225:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3145 00000100:00080000:2.0:1775767256.711227:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767256.711230:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767256.711234:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767256.711241:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742042880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767256.711259:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767256.711302:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767256/real 1775767256] req@ffff90c2034bf480 x1862023742042496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767256.711320:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023742042496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767256.711330:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767256.711335:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767256.711339:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742042496:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767256.711357:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767256.711366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767256.711384:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767256.711393:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767256/real 1775767256] req@ffff90c2034be300 x1862023742042624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767256.711402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023742042624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767256.711408:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767256.711410:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767256.711413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767256.711416:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767256.711426:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767256.711432:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767256/real 1775767256] req@ffff90c2034bca80 x1862023742042752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767256.711440:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742042752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767256.711457:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767256.711461:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767256.711464:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767256.711472:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042880:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767256.711494:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767256.711505:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767256/real 1775767256] req@ffff90c2034bea00 x1862023742042880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767281 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767256.711518:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742042880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767281 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767256.711534:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767256.711536:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767256.711538:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742042880:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767257.735075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767258.759105:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767259.783107:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767260.807065:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767261.831047:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767261.831057:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767261.831063:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767261.831074:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3150 00000100:00080000:2.0:1775767261.831086:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767261.831092:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767261.831127:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd500 x1862023742043008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767261.831152:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767261.831155:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767261.831157:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767261.831160:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3150 00000100:00080000:2.0:1775767261.831162:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767261.831164:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767261.831172:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767261.831179:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bfb80 x1862023742043136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767261.831187:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767261.831190:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767261.831191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767261.831194:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3150 00000100:00080000:2.0:1775767261.831197:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767261.831200:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767261.831203:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767261.831212:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742043264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767261.831218:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742043008:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775767261.831221:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767261.831227:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767261.831228:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767261.831232:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3150 00000100:00080000:2.0:1775767261.831234:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767261.831237:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767261.831240:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767261.831248:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742043392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767261.831291:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767261.831326:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767261/real 1775767261] req@ffff90c2034bd500 x1862023742043008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1775767261.831336:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767261.831341:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd500 x1862023742043008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767261.831350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767261.831354:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767261.831356:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742043008:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767261.831361:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767261.831381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767261.831389:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767261/real 1775767261] req@ffff90c2034bfb80 x1862023742043136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767261.831396:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bfb80 x1862023742043136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767261.831402:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767261.831404:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767261.831406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767261.831424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767261.831434:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767261.831440:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767261/real 1775767261] req@ffff90c2034bea00 x1862023742043264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767261.831447:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742043264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767261.831452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767261.831454:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767261.831456:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767261.831459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043392:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767261.831468:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767261.831473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767261/real 1775767261] req@ffff90c2034bca80 x1862023742043392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767261.831480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742043392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767286 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767261.831486:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767261.831488:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767261.831490:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043392:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767262.855025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767263.879132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767264.903022:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767265.927069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767266.951080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767266.951092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767266.951100:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767266.951112:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3155 00000100:00080000:1.0:1775767266.951127:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767266.951136:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767266.951175:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767266.951193:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742043520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767266.951231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767266.951236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767266.951239:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767266.951244:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3155 00000100:00080000:1.0:1775767266.951249:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767266.951255:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767266.951269:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767266.951289:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742043648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767266.951305:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767266.951310:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767266.951312:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767266.951316:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3155 00000100:00080000:1.0:1775767266.951322:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775767266.951324:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742043520:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767266.951330:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767266.951340:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767266.951353:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742043776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767266.951370:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767266.951374:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767266.951376:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767266.951380:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3155 00000100:00080000:1.0:1775767266.951396:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767266.951400:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767266.951405:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:0.0:1775767266.951416:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767266.951420:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742043904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767266.951472:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767266/real 1775767266] req@ffff90c203496d80 x1862023742043520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767266.951500:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742043520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767266.951515:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767266.951521:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767266.951527:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742043520:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767266.951538:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043648:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767266.951567:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767266.951580:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767266/real 1775767266] req@ffff90c203497b80 x1862023742043648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767266.951591:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742043648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767266.951599:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767266.951602:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767266.951605:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767266.951630:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043776:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767266.951645:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767266.951652:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767266/real 1775767266] req@ffff90c203496300 x1862023742043776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767266.951662:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742043776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767266.951674:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767266.951677:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767266.951680:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767266.951685:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043904:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767266.951698:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767266.951704:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767266/real 1775767266] req@ffff90c203496a00 x1862023742043904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767266.951713:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742043904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767291 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767266.951722:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767266.951725:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767266.951727:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742043904:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767267.975111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767268.999120:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767270.023107:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767271.047103:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767272.071018:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767272.071030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767272.071037:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767272.071050:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3160 00000100:00080000:3.0:1775767272.071065:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775767272.071071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767272.071074:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767272.071129:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742044032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767272.071162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767272.071166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767272.071168:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767272.071174:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3160 00000100:00080000:3.0:1775767272.071180:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767272.071184:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767272.071190:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742044032:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775767272.071196:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767272.071214:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742044160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767272.071226:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767272.071229:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767272.071231:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767272.071237:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3160 00000100:00080000:3.0:1775767272.071241:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767272.071244:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767272.071250:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767272.071265:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742044288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767272.071268:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767272.071280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767272.071284:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767272.071297:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767272.071307:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3160 00000100:00100000:0.0:1775767272.071312:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767272/real 1775767272] req@ffff90c218da0380 x1862023742044032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767272.071314:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767272.071324:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767272.071331:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767272.071344:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742044032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767272.071348:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742044416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767272.071372:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767272.071380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767272.071387:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742044032:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767272.071417:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767272.071443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767272.071457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767272/real 1775767272] req@ffff90c218da0700 x1862023742044160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767272.071471:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742044160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767272.071480:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767272.071483:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767272.071487:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767272.071494:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767272.071515:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767272.071526:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767272/real 1775767272] req@ffff90c218da3480 x1862023742044288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767272.071539:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742044288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767272.071547:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767272.071550:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767272.071553:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767272.071560:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044416:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767272.071580:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767272.071591:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767272/real 1775767272] req@ffff90c218da0e00 x1862023742044416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767272.071603:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742044416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767272.071614:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767272.071617:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767272.071621:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044416:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767273.095075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767274.119093:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767275.143080:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767276.167072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767277.191071:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767277.191080:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767277.191084:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767277.191093:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3165 00000100:00080000:3.0:1775767277.191104:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:0.0:1775767277.191107:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767277.191114:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767277.191163:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742044544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767277.191192:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767277.191196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767277.191198:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767277.191211:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3165 00000100:00080000:3.0:1775767277.191218:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767277.191223:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767277.191231:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742044544:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775767277.191234:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767277.191251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742044672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767277.191260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767277.191263:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767277.191264:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767277.191268:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3165 00000100:00080000:3.0:1775767277.191271:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767277.191274:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767277.191277:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767277.191285:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742044800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767277.191292:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767277.191294:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767277.191295:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767277.191298:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3165 00000100:00080000:3.0:1775767277.191300:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767277.191302:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767277.191306:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767277.191313:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742044928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767277.191346:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767277.191396:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767277/real 1775767277] req@ffff90c218da2d80 x1862023742044544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767277.191414:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742044544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767277.191424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767277.191427:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767277.191431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742044544:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767277.191457:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767277.191481:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767277.191495:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767277/real 1775767277] req@ffff90c218da3800 x1862023742044672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767277.191514:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742044672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767277.191521:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767277.191523:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767277.191526:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767277.191530:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767277.191547:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767277.191554:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767277/real 1775767277] req@ffff90c218da1880 x1862023742044800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767277.191562:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742044800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767277.191567:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767277.191569:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767277.191572:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767277.191575:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044928:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767277.191586:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767277.191591:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767277/real 1775767277] req@ffff90c218da0e00 x1862023742044928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767277.191598:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742044928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767277.191611:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767277.191613:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767277.191615:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742044928:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767278.215060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767279.239100:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767280.263037:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767281.287122:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767282.311022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767282.311036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:3.0:1775767282.311039:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767282.311043:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767282.311059:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3170 00000100:00080000:0.0:1775767282.311080:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767282.311089:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767282.311138:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742045056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767282.311171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767282.311176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767282.311178:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767282.311182:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3170 00000100:00080000:0.0:1775767282.311187:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767282.311191:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767282.311203:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767282.311219:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742045184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767282.311224:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742045056:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767282.311244:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767282.311249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767282.311251:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767282.311257:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3170 00000100:00080000:0.0:1775767282.311262:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767282.311266:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767282.311272:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767282.311285:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742045312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767282.311297:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767282.311302:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767282.311305:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767282.311308:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767282.311315:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3170 00000100:00080000:0.0:1775767282.311320:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767282.311325:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767282.311332:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767282.311346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767282/real 1775767282] req@ffff90c203639500 x1862023742045056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767282.311350:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742045440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767282.311380:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742045056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767282.311390:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767282.311395:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767282.311398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742045056:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767282.311432:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045184:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767282.311456:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767282.311470:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767282/real 1775767282] req@ffff90c20363a680 x1862023742045184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767282.311484:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742045184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767282.311495:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767282.311498:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767282.311502:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767282.311509:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767282.311532:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767282.311544:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767282/real 1775767282] req@ffff90c20363b100 x1862023742045312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767282.311557:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742045312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767282.311567:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767282.311571:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767282.311575:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767282.311581:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045440:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767282.311604:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767282.311616:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767282/real 1775767282] req@ffff90c20363bb80 x1862023742045440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767282.311629:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742045440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767282.311641:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767282.311644:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767282.311649:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045440:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767283.335136:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767284.359025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767285.383146:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767286.407136:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767287.431025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767287.431039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767287.431046:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:1.0:1775767287.431063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767287.431064:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3175 00000100:00080000:0.0:1775767287.431083:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767287.431092:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767287.431141:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742045568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767287.431166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767287.431169:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767287.431171:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767287.431174:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3175 00000100:00080000:0.0:1775767287.431176:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767287.431179:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767287.431189:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767287.431199:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742045696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767287.431205:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742045568:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767287.431213:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767287.431216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767287.431218:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767287.431221:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3175 00000100:00080000:0.0:1775767287.431223:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767287.431225:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767287.431228:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767287.431236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742045824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767287.431248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767287.431251:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767287.431252:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767287.431255:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3175 00000100:00080000:0.0:1775767287.431257:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767287.431259:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767287.431262:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767287.431272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742045952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767287.431281:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767287.431323:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767287/real 1775767287] req@ffff90c203638e00 x1862023742045568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767287.431339:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742045568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767287.431347:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767287.431351:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767287.431355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742045568:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767287.431377:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767287.431391:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767287.431399:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767287/real 1775767287] req@ffff90c20363b480 x1862023742045696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767287.431407:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742045696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767287.431413:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767287.431414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767287.431416:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767287.431420:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767287.431431:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767287.431437:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767287/real 1775767287] req@ffff90c203639f80 x1862023742045824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767287.431445:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742045824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767287.431450:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767287.431451:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767287.431453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767287.431456:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045952:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767287.431466:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767287.431471:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767287/real 1775767287] req@ffff90c20363b800 x1862023742045952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767287.431477:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742045952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767287.431485:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767287.431487:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767287.431489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742045952:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767288.455079:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767289.479088:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767290.503076:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767291.527077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767292.551031:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767292.551041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767292.551046:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767292.551056:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3181 00000100:00080000:0.0:1775767292.551068:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767292.551075:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767292.551120:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742046080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767292.551120:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767292.551159:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767292.551163:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767292.551164:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767292.551167:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3181 00000100:00080000:0.0:1775767292.551171:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767292.551174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767292.551183:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767292.551196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742046208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767292.551205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767292.551210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767292.551211:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742046080:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767292.551212:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767292.551220:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3181 00000100:00080000:0.0:1775767292.551222:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767292.551224:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767292.551227:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767292.551236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742046336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767292.551243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767292.551246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767292.551247:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767292.551250:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3181 00000100:00080000:0.0:1775767292.551252:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767292.551254:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767292.551257:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767292.551264:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742046464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767292.551316:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767292.551356:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767292.551376:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046464:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767293.575062:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767294.599039:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767295.623023:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767295.623104:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775767295.623136:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767295.623150:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767295.623185:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767295.623194:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767295.623203:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767295.623234:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767292/real 1775767295] req@ffff90c20363b800 x1862023742046208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767295.623260:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742046208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767295.623273:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767295.623278:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767295.623283:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767295.623291:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767292/real 1775767295] req@ffff90c203639f80 x1862023742046336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767295.623305:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742046336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767295.623314:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767295.623318:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767295.623321:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767295.623329:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767292/real 1775767295] req@ffff90c20363b480 x1862023742046464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767295.623343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742046464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767295.623353:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767295.623356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767295.623360:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046464:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767295.623368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767292/real 1775767295] req@ffff90c203638380 x1862023742046080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767295.623381:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742046080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767295.623389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767295.623393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767295.623402:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742046080:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775767296.647083:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767297.670991:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767297.670999:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767297.671003:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767297.671011:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3186 00000100:00080000:1.0:1775767297.671020:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767297.671027:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767297.671051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767297.671061:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742046592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767297.671083:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767297.671086:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767297.671087:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767297.671090:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3186 00000100:00080000:1.0:1775767297.671092:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767297.671094:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767297.671101:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767297.671108:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742046720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767297.671115:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767297.671118:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767297.671119:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767297.671121:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3186 00000100:00080000:1.0:1775767297.671123:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767297.671125:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767297.671128:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767297.671138:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742046848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767297.671139:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742046592:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767297.671157:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767297.671160:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767297.671162:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767297.671168:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3186 00000100:00080000:1.0:1775767297.671173:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767297.671178:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767297.671184:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767297.671196:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742046976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767297.671212:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767297.671256:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767297/real 1775767297] req@ffff90c203496680 x1862023742046592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767297.671275:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742046592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767297.671286:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767297.671291:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767297.671295:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742046592:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767297.671303:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767297.671326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767297.671339:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767297/real 1775767297] req@ffff90c203494a80 x1862023742046720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767297.671351:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742046720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767297.671360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767297.671363:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767297.671366:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767297.671390:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767297.671412:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767297.671422:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767297/real 1775767297] req@ffff90c203494700 x1862023742046848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767297.671433:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742046848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767297.671441:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767297.671450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767297.671454:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767297.671460:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046976:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767297.671481:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767297.671492:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767297/real 1775767297] req@ffff90c203495500 x1862023742046976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767322 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767297.671505:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742046976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767322 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767297.671523:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767297.671526:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767297.671530:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742046976:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767298.695033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767299.719007:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767300.743104:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767301.766987:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767302.790979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767302.790990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767302.790995:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767302.791007:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3191 00000100:00080000:0.0:1775767302.791021:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775767302.791028:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767302.791030:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767302.791091:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742047104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767302.791116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767302.791120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767302.791121:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767302.791124:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3191 00000100:00080000:0.0:1775767302.791127:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767302.791131:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767302.791141:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767302.791153:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742047232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767302.791162:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742047104:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767302.791163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767302.791171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767302.791173:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767302.791177:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3191 00000100:00080000:0.0:1775767302.791180:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767302.791182:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767302.791185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767302.791193:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742047360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767302.791201:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767302.791204:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767302.791205:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767302.791209:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3191 00000100:00080000:0.0:1775767302.791211:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767302.791213:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767302.791216:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767302.791222:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742047488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767302.791260:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767302.791308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767302/real 1775767302] req@ffff90c203638e00 x1862023742047104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767302.791331:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742047104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767302.791343:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767302.791348:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767302.791353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742047104:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767302.791382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767302.791406:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767302.791418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767302/real 1775767302] req@ffff90c20363bb80 x1862023742047232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767302.791432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742047232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767302.791442:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767302.791445:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767302.791450:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767302.791456:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767302.791478:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767302.791490:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767302/real 1775767302] req@ffff90c20363b100 x1862023742047360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767302.791514:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742047360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767302.791523:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767302.791526:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767302.791532:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767302.791539:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047488:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767302.791561:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767302.791608:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767302/real 1775767302] req@ffff90c20363a680 x1862023742047488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767327 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767302.791622:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742047488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767327 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767302.791634:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767302.791638:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767302.791642:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047488:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767303.815081:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767304.839077:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767305.863074:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767306.887024:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767307.911027:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767307.911038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767307.911043:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767307.911054:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3196 00000100:00080000:3.0:1775767307.911067:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767307.911075:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775767307.911090:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767307.911130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742047616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767307.911158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767307.911162:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767307.911164:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767307.911168:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3196 00000100:00080000:3.0:1775767307.911173:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767307.911178:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767307.911190:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767307.911202:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742047616:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767307.911214:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742047744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767307.911225:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767307.911228:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767307.911229:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767307.911233:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3196 00000100:00080000:3.0:1775767307.911237:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767307.911240:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767307.911245:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767307.911254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742047872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767307.911260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767307.911263:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767307.911264:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767307.911267:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3196 00000100:00080000:3.0:1775767307.911269:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767307.911272:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767307.911274:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767307.911280:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767307.911285:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742048000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767307.911329:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767307/real 1775767307] req@ffff90c218da3480 x1862023742047616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767307.911351:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742047616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767307.911364:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767307.911369:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767307.911374:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742047616:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767307.911403:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047744:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767307.911425:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767307.911437:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767307/real 1775767307] req@ffff90c218da0700 x1862023742047744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767307.911450:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742047744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767307.911459:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767307.911462:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767307.911466:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767307.911473:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047872:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767307.911489:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767307.911498:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767307/real 1775767307] req@ffff90c218da0380 x1862023742047872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767307.911547:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742047872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767307.911556:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767307.911560:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767307.911563:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742047872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767307.911571:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048000:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767307.911591:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767307.911601:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767307/real 1775767307] req@ffff90c218da0a80 x1862023742048000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767307.911624:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742048000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767332 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767307.911636:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767307.911640:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767307.911644:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048000:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767308.935098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767309.959108:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767310.983097:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767312.007109:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767313.030999:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767313.031011:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767313.031016:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775767313.031026:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767313.031027:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3201 00000100:00080000:0.0:1775767313.031041:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767313.031048:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767313.031095:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742048128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767313.031122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767313.031126:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767313.031129:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767313.031133:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3201 00000100:00080000:0.0:1775767313.031137:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767313.031142:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767313.031155:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767313.031159:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742048128:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767313.031174:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742048256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767313.031186:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767313.031189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767313.031191:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767313.031196:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3201 00000100:00080000:0.0:1775767313.031200:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767313.031205:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767313.031210:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767313.031223:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742048384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767313.031233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767313.031236:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767313.031240:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767313.031242:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767313.031248:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3201 00000100:00080000:0.0:1775767313.031253:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767313.031257:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767313.031263:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767313.031278:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742048512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767313.031283:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767313/real 1775767313] req@ffff90c203639500 x1862023742048128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767313.031311:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742048128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767313.031320:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767313.031324:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767313.031328:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742048128:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767313.031350:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767313.031365:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767313.031373:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767313/real 1775767313] req@ffff90c203638000 x1862023742048256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767313.031382:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742048256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767313.031388:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767313.031390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767313.031392:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767313.031395:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767313.031405:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767313.031410:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767313/real 1775767313] req@ffff90c203639180 x1862023742048384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767313.031424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742048384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767313.031429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767313.031431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767313.031433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767313.031437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048512:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767313.031448:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767313.031453:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767313/real 1775767313] req@ffff90c20363aa00 x1862023742048512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767313.031460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742048512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767313.031490:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767313.031492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767313.031494:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048512:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767314.055106:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767315.079058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767316.103058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767317.127008:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767318.150984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767318.150992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767318.150996:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767318.151004:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3206 00000100:00080000:0.0:1775767318.151015:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767318.151022:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767318.151065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742048640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775767318.151083:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767318.151095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767318.151100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767318.151102:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767318.151108:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3206 00000100:00080000:0.0:1775767318.151113:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767318.151119:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767318.151127:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742048640:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775767318.151129:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767318.151146:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742048768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767318.151155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767318.151158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767318.151160:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767318.151163:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3206 00000100:00080000:0.0:1775767318.151166:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767318.151169:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767318.151172:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767318.151182:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742048896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767318.151185:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767318.151193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767318.151196:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767318.151197:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767318.151208:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3206 00000100:00080000:0.0:1775767318.151211:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767318.151213:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767318.151217:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767318.151222:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767318/real 1775767318] req@ffff90c203638700 x1862023742048640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767318.151227:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742049024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767318.151253:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742048640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767318.151262:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767318.151265:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767318.151268:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742048640:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767318.151285:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767318.151300:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767318.151307:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767318/real 1775767318] req@ffff90c20363aa00 x1862023742048768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767318.151315:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742048768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767318.151321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767318.151323:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767318.151325:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767318.151329:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048896:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767318.151339:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767318.151354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767318/real 1775767318] req@ffff90c203639180 x1862023742048896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767318.151362:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742048896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767318.151367:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767318.151370:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767318.151373:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742048896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767318.151380:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049024:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767318.151398:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767318.151408:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767318/real 1775767318] req@ffff90c203638000 x1862023742049024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767318.151450:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742049024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767318.151462:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767318.151465:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767318.151467:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049024:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767319.175055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767320.199073:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767321.223120:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767322.247101:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767323.271040:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767323.271050:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767323.271055:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:0.0:1775767323.271060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767323.271067:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3211 00000100:00080000:1.0:1775767323.271082:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767323.271090:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767323.271144:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742049152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767323.271175:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767323.271178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767323.271181:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767323.271186:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3211 00000100:00080000:1.0:1775767323.271190:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767323.271195:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767323.271207:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742049152:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775767323.271208:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767323.271229:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742049280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767323.271239:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767323.271243:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767323.271245:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767323.271249:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3211 00000100:00080000:1.0:1775767323.271252:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767323.271257:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767323.271261:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767323.271272:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742049408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767323.271281:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767323.271284:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767323.271286:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767323.271292:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767323.271293:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3211 00000100:00080000:1.0:1775767323.271300:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767323.271304:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767323.271310:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767323.271323:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742049536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767323.271347:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767323/real 1775767323] req@ffff90c203495180 x1862023742049152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767323.271366:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742049152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767323.271389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767323.271394:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767323.271398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742049152:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767323.271428:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767323.271450:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767323.271460:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767323/real 1775767323] req@ffff90c203495500 x1862023742049280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767323.271468:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742049280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767323.271474:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767323.271477:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767323.271479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767323.271483:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767323.271494:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767323.271500:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767323/real 1775767323] req@ffff90c203494700 x1862023742049408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767323.271507:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742049408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767323.271517:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767323.271519:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767323.271521:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767323.271524:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049536:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767323.271534:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767323.271539:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767323/real 1775767323] req@ffff90c203494a80 x1862023742049536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767323.271545:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742049536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767323.271552:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767323.271554:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767323.271556:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049536:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767324.295075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767325.319125:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767326.343062:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767327.367063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767328.390999:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767328.391014:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767328.391020:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767328.391040:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3216 00000100:00080000:1.0:1775767328.391056:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767328.391065:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767328.391083:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767328.391136:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742049664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767328.391173:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767328.391177:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767328.391179:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767328.391184:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3216 00000100:00080000:1.0:1775767328.391189:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767328.391193:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767328.391207:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767328.391218:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742049664:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767328.391226:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742049792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767328.391240:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767328.391244:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767328.391247:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767328.391253:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3216 00000100:00080000:1.0:1775767328.391258:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767328.391263:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767328.391270:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767328.391283:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742049920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767328.391294:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767328.391298:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767328.391300:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767328.391311:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3216 00000800:00000100:2.0:1775767328.391313:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767328.391318:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767328.391325:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767328.391348:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767328.391362:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742050048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767328.391392:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767328/real 1775767328] req@ffff90c203496680 x1862023742049664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767328.391408:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742049664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767328.391416:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767328.391419:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767328.391422:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742049664:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767328.391443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767328.391460:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767328.391469:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767328/real 1775767328] req@ffff90c203496a00 x1862023742049792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767328.391477:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742049792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767328.391482:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767328.391485:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767328.391488:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767328.391495:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767328.391515:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767328.391527:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767328/real 1775767328] req@ffff90c203496300 x1862023742049920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767328.391540:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742049920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767328.391547:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767328.391551:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767328.391554:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742049920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767328.391560:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050048:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767328.391579:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767328.391588:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767328/real 1775767328] req@ffff90c203497b80 x1862023742050048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767328.391600:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742050048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767328.391609:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767328.391613:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767328.391616:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050048:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767329.415070:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767330.439088:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767331.463112:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767332.487104:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767333.511030:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767333.511038:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767333.511043:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767333.511052:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3221 00000100:00080000:3.0:1775767333.511064:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775767333.511071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767333.511073:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767333.511110:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742050176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767333.511136:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767333.511139:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767333.511140:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767333.511143:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3221 00000100:00080000:3.0:1775767333.511146:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767333.511148:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767333.511156:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767333.511165:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742050304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767333.511177:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767333.511187:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742050176:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767333.511190:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767333.511195:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767333.511206:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3221 00000100:00080000:3.0:1775767333.511210:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767333.511215:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767333.511222:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767333.511235:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742050432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767333.511242:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767333.511245:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767333.511253:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767333.511256:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3221 00000100:00080000:3.0:1775767333.511309:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767333.511310:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767333.511320:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767333.511328:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767333.511343:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742050560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767333.511373:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767333/real 1775767333] req@ffff90c218da1f80 x1862023742050176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767333.511398:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742050176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767333.511411:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767333.511416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767333.511421:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742050176:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767333.511452:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050304:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767333.511480:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767333.511494:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767333/real 1775767333] req@ffff90c218da0000 x1862023742050304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767333.511506:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742050304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767333.511522:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767333.511525:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767333.511528:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767333.511535:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050432:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767333.511554:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767333.511565:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767333/real 1775767333] req@ffff90c218da3100 x1862023742050432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767333.511578:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742050432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767333.511584:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767333.511586:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767333.511589:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767333.511593:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050560:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767333.511609:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767333.511616:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767333/real 1775767333] req@ffff90c218da0a80 x1862023742050560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767333.511623:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742050560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767333.511630:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767333.511633:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767333.511635:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050560:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767334.535067:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767335.559125:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767336.582988:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767337.607132:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767338.630986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767338.630999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767338.631006:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767338.631024:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3227 00010000:00010000:3.0:1775767338.631029:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767338.631038:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767338.631047:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767338.631096:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742050688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767338.631126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767338.631130:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767338.631133:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767338.631137:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3227 00000100:00080000:0.0:1775767338.631141:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767338.631145:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767338.631157:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767338.631168:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742050688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767338.631173:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742050816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767338.631187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767338.631191:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767338.631194:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767338.631199:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3227 00000100:00080000:0.0:1775767338.631203:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767338.631207:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767338.631213:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767338.631225:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742050944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767338.631262:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767338.631267:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767338.631269:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767338.631272:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767338.631279:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3227 00000100:00080000:0.0:1775767338.631283:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767338.631287:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767338.631293:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767338.631309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742051072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767338.631316:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767338/real 1775767338] req@ffff90c203639500 x1862023742050688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767338.631341:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742050688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767338.631354:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767338.631359:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767338.631364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742050688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767338.631389:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767338.631410:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767338.631421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767338/real 1775767338] req@ffff90c20363a680 x1862023742050816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767338.631434:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742050816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767338.631444:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767338.631447:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767338.631451:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767338.631457:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767338.631472:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767338.631482:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767338/real 1775767338] req@ffff90c20363b100 x1862023742050944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767338.631493:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742050944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767338.631502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767338.631506:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767338.631509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742050944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767338.631516:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051072:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767338.631535:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767338.631545:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767338/real 1775767338] req@ffff90c20363bb80 x1862023742051072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767363 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767338.631567:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742051072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767363 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767338.631579:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767338.631583:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767338.631586:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051072:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767339.655067:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767340.678993:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767341.703110:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767342.727136:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767343.751088:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767343.751109:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767343.751116:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767343.751131:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3232 00000100:00080000:2.0:1775767343.751149:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767343.751159:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767343.751245:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023742051200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767343.751294:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767343.751299:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767343.751302:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767343.751307:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3232 00000100:00080000:2.0:1775767343.751312:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767343.751318:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767343.751331:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767343.751351:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023742051328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767343.751361:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742051200:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775767343.751373:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767343.751379:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767343.751382:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767343.751389:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3232 00000100:00080000:2.0:1775767343.751395:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767343.751399:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767343.751406:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767343.751421:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023742051456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767343.751429:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767343.751433:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767343.751436:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767343.751440:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3232 00000100:00080000:2.0:1775767343.751444:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767343.751447:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:0.0:1775767343.751453:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:2.0:1775767343.751453:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767343.751467:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023742051584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767343.751523:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767343/real 1775767343] req@ffff90c2034be300 x1862023742051200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767343.751552:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023742051200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767343.751568:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767343.751576:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00010000:00010000:3.0:1775767343.751576:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767343.751586:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742051200:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767343.751618:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767343.751644:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767343.751657:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767343/real 1775767343] req@ffff90c2034bf480 x1862023742051328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767343.751668:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023742051328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767343.751676:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767343.751679:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767343.751683:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767343.751687:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767343.751701:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767343.751708:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767343/real 1775767343] req@ffff90c2034bce00 x1862023742051456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767343.751718:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023742051456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767343.751725:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767343.751727:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767343.751730:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767343.751734:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051584:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767343.751747:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767343.751753:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767343/real 1775767343] req@ffff90c2034bc380 x1862023742051584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767368 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767343.751763:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023742051584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767368 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767343.751772:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767343.751775:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767343.751778:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051584:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767344.775137:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767345.799074:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767346.823130:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767347.847162:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767348.871052:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767348.871060:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767348.871065:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767348.871073:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3237 00000100:00080000:0.0:1775767348.871083:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767348.871091:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767348.871132:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742051712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775767348.871192:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767348.871193:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767348.871199:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767348.871201:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767348.871205:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3237 00000100:00080000:0.0:1775767348.871209:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767348.871214:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767348.871223:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767348.871235:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742051840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767348.871244:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767348.871250:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767348.871250:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742051712:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767348.871253:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767348.871266:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3237 00000100:00080000:0.0:1775767348.871273:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767348.871276:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767348.871278:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767348.871289:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742051968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767348.871298:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767348.871300:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767348.871308:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767348.871311:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3237 00000100:00080000:0.0:1775767348.871314:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767348.871316:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767348.871320:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767348.871328:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742052096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767348.871362:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767348.871413:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767348/real 1775767348] req@ffff90c203638e00 x1862023742051712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767348.871436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742051712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767348.871449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767348.871454:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767348.871459:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742051712:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767348.871491:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051840:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767348.871516:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767348.871530:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767348/real 1775767348] req@ffff90c203638380 x1862023742051840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767348.871545:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742051840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767348.871555:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767348.871560:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767348.871563:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767348.871570:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767348.871606:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767348.871619:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767348/real 1775767348] req@ffff90c20363b480 x1862023742051968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767348.871632:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742051968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767348.871641:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767348.871645:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767348.871649:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742051968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767348.871655:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052096:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767348.871676:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767348.871686:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767348/real 1775767348] req@ffff90c203639f80 x1862023742052096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767348.871698:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742052096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767373 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767348.871710:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767348.871714:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767348.871717:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052096:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767349.895083:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767350.919073:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767351.943102:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767352.967195:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767353.990977:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767353.990988:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767353.990994:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767353.991007:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3242 00000100:00080000:0.0:1775767353.991022:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767353.991031:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767353.991040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767353.991085:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742052224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767353.991133:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767353.991136:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767353.991138:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767353.991142:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3242 00000100:00080000:0.0:1775767353.991145:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767353.991149:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767353.991159:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767353.991165:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742052224:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767353.991177:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742052352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767353.991186:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767353.991189:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767353.991191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767353.991194:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3242 00000100:00080000:0.0:1775767353.991197:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767353.991199:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767353.991202:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767353.991210:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742052480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767353.991216:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767353.991219:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767353.991220:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767353.991223:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3242 00000100:00080000:0.0:1775767353.991227:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767353.991229:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767353.991233:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767353.991239:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767353.991243:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742052608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767353.991281:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767353/real 1775767353] req@ffff90c20363b800 x1862023742052224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767353.991297:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742052224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767353.991305:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767353.991309:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767353.991312:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742052224:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767353.991334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767353.991349:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767353.991359:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767353/real 1775767353] req@ffff90c203639f80 x1862023742052352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767353.991371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742052352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767353.991380:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767353.991383:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767353.991387:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767353.991393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767353.991414:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767353.991425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767353/real 1775767353] req@ffff90c20363b480 x1862023742052480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767353.991444:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742052480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767353.991450:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767353.991452:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767353.991454:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767353.991459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052608:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767353.991474:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767353.991480:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767353/real 1775767353] req@ffff90c203638380 x1862023742052608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767353.991488:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742052608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767378 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767353.991496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767353.991497:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767353.991499:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052608:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767355.015226:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767356.039018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767357.063161:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767358.087008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767359.111002:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767359.111012:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767359.111017:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767359.111028:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3247 00000100:00080000:0.0:1775767359.111068:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767359.111078:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767359.111094:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767359.111133:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742052736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767359.111156:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767359.111159:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767359.111161:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767359.111164:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3247 00000100:00080000:0.0:1775767359.111168:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767359.111171:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767359.111181:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767359.111192:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742052864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767359.111193:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742052736:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767359.111207:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767359.111210:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767359.111211:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767359.111214:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3247 00000100:00080000:0.0:1775767359.111217:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767359.111219:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767359.111222:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767359.111230:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742052992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767359.111236:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767359.111238:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767359.111239:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767359.111242:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3247 00000100:00080000:0.0:1775767359.111245:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767359.111248:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767359.111251:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767359.111258:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742053120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767359.111307:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767359.111354:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767359.111371:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053120:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767360.135114:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767361.159237:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767362.183167:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775767362.183270:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775767362.183336:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775767362.183354:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767362.183397:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767362.183407:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767362.183415:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767362.183473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767359/real 1775767362] req@ffff90c20363bb80 x1862023742052864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767362.183520:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742052864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767362.183537:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767362.183545:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767362.183550:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767362.183562:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767359/real 1775767362] req@ffff90c20363b100 x1862023742052992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767362.183581:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742052992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767362.183592:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767362.183598:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767362.183602:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742052992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767362.183612:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767359/real 1775767362] req@ffff90c20363a680 x1862023742053120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767362.183628:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742053120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767362.183641:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767362.183646:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767362.183650:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053120:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767362.183659:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767359/real 1775767362] req@ffff90c203638e00 x1862023742052736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767362.183674:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742052736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767362.183683:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767362.183687:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767362.183691:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742052736:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767363.207262:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767364.231086:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767364.231099:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767364.231106:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767364.231121:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3252 00000100:00080000:0.0:1775767364.231138:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767364.231149:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767364.231204:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742053248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775767364.231206:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767364.231237:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767364.231241:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767364.231243:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767364.231250:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3252 00000100:00080000:0.0:1775767364.231255:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767364.231260:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767364.231276:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767364.231285:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742053248:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767364.231305:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742053376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767364.231319:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767364.231324:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767364.231326:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767364.231330:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3252 00000100:00080000:0.0:1775767364.231335:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767364.231340:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767364.231347:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767364.231365:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742053504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767364.231379:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767364.231380:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767364.231386:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767364.231389:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767364.231398:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3252 00000100:00080000:0.0:1775767364.231403:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767364.231407:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767364.231414:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767364.231434:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742053632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767364.231434:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767364/real 1775767364] req@ffff90c203639500 x1862023742053248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767364.231506:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742053248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767364.231521:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767364.231528:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767364.231532:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742053248:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767364.231562:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767364.231589:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767364.231603:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767364/real 1775767364] req@ffff90c203638000 x1862023742053376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767364.231616:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742053376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767364.231625:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767364.231630:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767364.231634:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767364.231642:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767364.231662:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767364.231673:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767364/real 1775767364] req@ffff90c203639180 x1862023742053504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767364.231686:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742053504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767364.231695:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767364.231698:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767364.231702:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767364.231710:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053632:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767364.231736:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767364.231747:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767364/real 1775767364] req@ffff90c20363aa00 x1862023742053632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767364.231758:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742053632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767364.231769:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767364.231773:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767364.231776:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053632:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767365.255182:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767366.279065:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767367.303032:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767368.327055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767369.351031:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767369.351041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767369.351047:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767369.351059:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3257 00010000:00010000:3.0:1775767369.351068:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767369.351073:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767369.351082:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767369.351128:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742053760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767369.351156:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767369.351159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767369.351160:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767369.351163:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3257 00000100:00080000:0.0:1775767369.351166:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767369.351168:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767369.351178:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767369.351189:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742053888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767369.351199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767369.351202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767369.351203:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767369.351207:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3257 00000100:00080000:0.0:1775767369.351215:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767369.351219:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767369.351219:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742053760:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775767369.351223:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767369.351236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742054016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767369.351242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767369.351244:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767369.351245:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767369.351249:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3257 00000100:00080000:0.0:1775767369.351251:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767369.351254:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767369.351258:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767369.351266:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742054144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767369.351307:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767369.351349:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767369/real 1775767369] req@ffff90c203638700 x1862023742053760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767369.351385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742053760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767369.351397:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767369.351402:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767369.351406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742053760:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767369.351431:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767369.351455:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767369.351468:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767369/real 1775767369] req@ffff90c20363aa00 x1862023742053888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767369.351480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742053888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767369.351488:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767369.351492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767369.351495:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742053888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767369.351502:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767369.351521:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767369.351531:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767369/real 1775767369] req@ffff90c203639180 x1862023742054016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767369.351551:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742054016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767369.351569:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767369.351572:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767369.351575:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767369.351582:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054144:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767369.351605:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767369.351617:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767369/real 1775767369] req@ffff90c203638000 x1862023742054144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767369.351630:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742054144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767369.351641:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767369.351645:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767369.351648:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054144:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767370.375060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767371.399087:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767372.423045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767373.447063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767374.471001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767374.471013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767374.471019:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767374.471031:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3262 00000100:00080000:3.0:1775767374.471061:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767374.471070:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767374.471077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767374.471117:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742054272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767374.471144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767374.471148:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767374.471149:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767374.471152:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3262 00000100:00080000:3.0:1775767374.471155:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767374.471158:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767374.471168:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767374.471179:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742054400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767374.471188:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767374.471190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767374.471192:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767374.471196:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742054272:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767374.471199:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3262 00000100:00080000:3.0:1775767374.471205:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767374.471208:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767374.471211:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767374.471222:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742054528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767374.471228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767374.471230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767374.471232:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767374.471235:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3262 00000100:00080000:3.0:1775767374.471238:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767374.471240:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767374.471243:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767374.471250:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742054656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767374.471282:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767374.471322:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767374/real 1775767374] req@ffff90c218da0380 x1862023742054272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767374.471341:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742054272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767374.471360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767374.471365:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767374.471369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742054272:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767374.471399:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054400:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767374.471419:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767374.471428:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767374/real 1775767374] req@ffff90c218da0700 x1862023742054400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767374.471438:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742054400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767374.471444:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767374.471446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767374.471448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767374.471452:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054528:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767374.471472:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767374.471479:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767374/real 1775767374] req@ffff90c218da3480 x1862023742054528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767374.471487:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742054528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767374.471492:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767374.471494:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767374.471496:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767374.471501:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054656:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767374.471512:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767374.471517:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767374/real 1775767374] req@ffff90c218da0e00 x1862023742054656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767374.471525:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742054656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767374.471532:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767374.471534:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767374.471535:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054656:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767375.495010:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767376.519125:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767377.543079:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767378.567096:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767379.591131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767379.591147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767379.591154:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767379.591170:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3268 00000100:00080000:0.0:1775767379.591187:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767379.591215:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767379.591295:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742054784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775767379.591302:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767379.591341:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767379.591346:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767379.591349:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767379.591355:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3268 00000100:00080000:0.0:1775767379.591360:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767379.591365:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767379.591379:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767379.591399:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742054912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767379.591415:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767379.591419:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742054784:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767379.591421:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767379.591427:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767379.591440:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3268 00000100:00080000:0.0:1775767379.591444:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767379.591448:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767379.591456:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767379.591474:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742055040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767379.591487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767379.591491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767379.591494:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767379.591499:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3268 00000100:00080000:0.0:1775767379.591505:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767379.591509:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767379.591514:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767379.591527:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742055168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767379.591553:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767379.591626:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767379/real 1775767379] req@ffff90c203639500 x1862023742054784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767379.591661:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742054784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767379.591681:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767379.591688:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767379.591697:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742054784:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767379.591753:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767379.591814:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767379.591831:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767379/real 1775767379] req@ffff90c203638e00 x1862023742054912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767379.591906:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742054912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767379.591918:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767379.591923:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767379.591928:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742054912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767379.591936:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767379.591976:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767379.591990:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767379/real 1775767379] req@ffff90c20363a680 x1862023742055040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767379.592006:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742055040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767379.592017:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767379.592022:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767379.592027:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767379.592035:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055168:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767379.592063:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767379.592075:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767379/real 1775767379] req@ffff90c20363b100 x1862023742055168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767379.592090:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742055168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767379.592144:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767379.592149:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767379.592155:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055168:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767380.615082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767381.639115:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767382.663072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767383.687045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767384.711029:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767384.711041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767384.711047:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767384.711059:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3273 00000100:00080000:0.0:1775767384.711073:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767384.711082:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767384.711089:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767384.711123:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742055296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767384.711155:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767384.711159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767384.711161:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767384.711165:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3273 00000100:00080000:0.0:1775767384.711169:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767384.711174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767384.711186:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767384.711201:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742055424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767384.711212:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742055296:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767384.711229:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767384.711232:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767384.711234:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767384.711241:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3273 00000100:00080000:0.0:1775767384.711247:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767384.711251:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767384.711259:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767384.711276:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742055552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767384.711287:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767384.711291:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767384.711293:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767384.711297:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3273 00000100:00080000:0.0:1775767384.711301:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767384.711310:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767384.711311:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:0.0:1775767384.711326:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767384.711340:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742055680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767384.711372:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767384/real 1775767384] req@ffff90c20363bb80 x1862023742055296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767384.711393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742055296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767384.711404:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767384.711408:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767384.711411:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742055296:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767384.711438:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055424:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767384.711456:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767384.711465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767384/real 1775767384] req@ffff90c203638380 x1862023742055424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767384.711473:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742055424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767384.711479:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767384.711481:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767384.711483:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767384.711489:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055552:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767384.711500:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767384.711505:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767384/real 1775767384] req@ffff90c20363b480 x1862023742055552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767384.711513:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742055552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767384.711518:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767384.711519:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767384.711521:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767384.711525:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055680:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767384.711535:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767384.711540:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767384/real 1775767384] req@ffff90c203639f80 x1862023742055680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767409 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767384.711546:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742055680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767409 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767384.711554:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767384.711556:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767384.711558:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055680:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767385.735132:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767386.759068:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767387.783040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767388.807115:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767389.831030:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767389.831041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767389.831048:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767389.831059:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3278 00000100:00080000:3.0:1775767389.831073:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767389.831080:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767389.831109:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767389.831129:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742055808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767389.831154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767389.831157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767389.831158:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767389.831162:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3278 00000100:00080000:3.0:1775767389.831164:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767389.831167:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767389.831177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767389.831186:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742055936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767389.831196:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742055808:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767389.831197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767389.831212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767389.831218:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767389.831224:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3278 00000100:00080000:3.0:1775767389.831228:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767389.831232:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767389.831238:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767389.831253:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742056064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767389.831263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767389.831268:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767389.831270:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767389.831274:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3278 00000100:00080000:3.0:1775767389.831278:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767389.831285:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767389.831285:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767389.831294:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767389.831309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742056192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767389.831330:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767389/real 1775767389] req@ffff90c218da1880 x1862023742055808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767389.831348:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742055808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767389.831357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767389.831361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767389.831365:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742055808:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767389.831387:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055936:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767389.831403:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767389.831411:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767389/real 1775767389] req@ffff90c218da3800 x1862023742055936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767389.831419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742055936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767389.831426:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767389.831428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767389.831430:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742055936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767389.831433:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767389.831443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767389.831448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767389/real 1775767389] req@ffff90c218da2d80 x1862023742056064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767389.831455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742056064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767389.831465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767389.831467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767389.831469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767389.831473:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056192:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767389.831482:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767389.831487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767389/real 1775767389] req@ffff90c218da0e00 x1862023742056192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767389.831493:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742056192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767414 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767389.831500:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767389.831501:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767389.831503:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056192:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767390.855045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767391.879114:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767392.903020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767393.927053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767394.950950:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767394.950960:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767394.950966:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767394.950976:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3283 00000100:00080000:2.0:1775767394.950989:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767394.950996:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767394.951040:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023742056320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767394.951071:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767394.951075:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767394.951076:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767394.951079:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3283 00000100:00080000:2.0:1775767394.951082:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767394.951084:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767394.951093:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767394.951104:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023742056448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767394.951113:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767394.951116:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767394.951117:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767394.951120:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3283 00000100:00080000:2.0:1775767394.951123:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767394.951126:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767394.951129:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767394.951134:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742056320:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767394.951139:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023742056576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767394.951156:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767394.951159:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767394.951160:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767394.951165:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3283 00000100:00080000:2.0:1775767394.951167:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767394.951171:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767394.951175:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767394.951182:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023742056704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767394.951213:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767394.951268:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767394/real 1775767394] req@ffff90c2034bd180 x1862023742056320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767394.951284:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023742056320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:1.0:1775767394.951290:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767394.951293:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767394.951298:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767394.951301:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742056320:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767394.951321:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767394.951337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767394.951345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767394/real 1775767394] req@ffff90c2034bdf80 x1862023742056448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767394.951353:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023742056448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767394.951359:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767394.951361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767394.951363:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767394.951367:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767394.951377:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767394.951383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767394/real 1775767394] req@ffff90c2034bf100 x1862023742056576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767394.951397:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023742056576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767394.951402:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767394.951404:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767394.951406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767394.951410:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056704:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767394.951420:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767394.951425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767394/real 1775767394] req@ffff90c2034bc700 x1862023742056704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767394.951432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023742056704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767419 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767394.951439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767394.951441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767394.951443:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056704:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767395.975143:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767396.999082:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767398.023078:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767399.047058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767400.071049:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767400.071057:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767400.071062:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767400.071071:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3288 00000100:00080000:0.0:1775767400.071084:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775767400.071091:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767400.071092:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767400.071136:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742056832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767400.071168:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767400.071170:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767400.071172:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767400.071175:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3288 00000100:00080000:0.0:1775767400.071178:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767400.071182:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767400.071191:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767400.071204:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742056960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767400.071216:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742056832:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767400.071217:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767400.071224:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767400.071228:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767400.071236:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3288 00000100:00080000:0.0:1775767400.071238:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767400.071240:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767400.071244:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767400.071252:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742057088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767400.071259:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767400.071261:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767400.071263:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767400.071267:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3288 00000100:00080000:0.0:1775767400.071270:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767400.071272:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767400.071275:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767400.071281:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742057216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767400.071339:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767400.071401:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767400/real 1775767400] req@ffff90c20363b800 x1862023742056832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767400.071423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742056832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767400.071435:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767400.071441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767400.071445:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742056832:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767400.071477:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056960:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767400.071502:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767400.071514:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767400/real 1775767400] req@ffff90c203639f80 x1862023742056960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767400.071526:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742056960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767400.071535:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767400.071539:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767400.071542:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742056960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767400.071550:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057088:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767400.071571:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767400.071589:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767400/real 1775767400] req@ffff90c20363b480 x1862023742057088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767400.071598:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742057088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767400.071604:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767400.071606:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767400.071609:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767400.071612:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057216:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767400.071625:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767400.071631:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767400/real 1775767400] req@ffff90c203638380 x1862023742057216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767400.071638:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742057216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767400.071646:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767400.071649:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767400.071652:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057216:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767401.095182:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767402.119035:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767403.143031:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767404.167081:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767405.191040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767405.191050:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767405.191056:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767405.191068:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3293 00000100:00080000:3.0:1775767405.191080:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767405.191087:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767405.191128:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767405.191133:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742057344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767405.191156:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767405.191159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767405.191161:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767405.191164:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3293 00000100:00080000:3.0:1775767405.191167:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767405.191171:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767405.191181:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767405.191190:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742057472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767405.191195:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742057344:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767405.191205:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767405.191208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767405.191209:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767405.191212:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3293 00000100:00080000:3.0:1775767405.191214:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767405.191216:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767405.191219:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767405.191228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742057600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767405.191234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767405.191236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767405.191237:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767405.191240:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3293 00000100:00080000:3.0:1775767405.191262:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767405.191265:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767405.191268:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767405.191275:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767405.191279:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742057728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767405.191318:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767405/real 1775767405] req@ffff90c218da3480 x1862023742057344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767405.191340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742057344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767405.191348:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767405.191352:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767405.191355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742057344:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767405.191377:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767405.191391:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767405.191398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767405/real 1775767405] req@ffff90c218da0700 x1862023742057472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767405.191406:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742057472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767405.191412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767405.191414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767405.191416:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767405.191419:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767405.191429:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767405.191434:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767405/real 1775767405] req@ffff90c218da0380 x1862023742057600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767405.191441:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742057600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767405.191450:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767405.191452:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767405.191455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767405.191458:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057728:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767405.191468:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767405.191472:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767405/real 1775767405] req@ffff90c218da0a80 x1862023742057728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767405.191487:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742057728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767405.191494:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767405.191496:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767405.191498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057728:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767406.215069:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767407.239115:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767408.263025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767409.287033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767410.310988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767410.310998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767410.311003:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767410.311013:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3298 00010000:00010000:1.0:1775767410.311020:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767410.311026:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767410.311035:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767410.311086:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742057856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767410.311114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767410.311116:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767410.311118:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767410.311121:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3298 00000100:00080000:0.0:1775767410.311125:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767410.311131:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767410.311142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767410.311149:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742057856:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767410.311163:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742057984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767410.311176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767410.311180:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767410.311182:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767410.311188:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3298 00000100:00080000:0.0:1775767410.311193:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767410.311197:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767410.311203:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767410.311218:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742058112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767410.311220:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767410.311230:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767410.311233:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767410.311234:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767410.311239:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3298 00000100:00080000:0.0:1775767410.311243:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767410.311245:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767410.311250:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767410.311261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742058240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767410.311273:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767410/real 1775767410] req@ffff90c20363bb80 x1862023742057856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767410.311291:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742057856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767410.311300:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767410.311304:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767410.311307:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742057856:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767410.311330:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767410.311346:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767410.311355:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767410/real 1775767410] req@ffff90c20363b100 x1862023742057984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767410.311363:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742057984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767410.311369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767410.311371:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767410.311373:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742057984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767410.311376:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767410.311387:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767410.311393:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767410/real 1775767410] req@ffff90c20363a680 x1862023742058112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767410.311400:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742058112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767410.311405:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767410.311407:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767410.311409:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767410.311413:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058240:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767410.311423:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767410.311427:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767410/real 1775767410] req@ffff90c203638e00 x1862023742058240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767410.311443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742058240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767410.311451:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767410.311453:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767410.311455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058240:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767411.335052:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767412.359050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767413.383112:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767414.407028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767415.430995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767415.431006:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767415.431012:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767415.431027:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3303 00000100:00080000:0.0:1775767415.431042:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767415.431052:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767415.431078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767415.431104:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742058368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767415.431136:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767415.431140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767415.431142:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767415.431148:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3303 00000100:00080000:0.0:1775767415.431152:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767415.431157:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767415.431169:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767415.431169:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742058368:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767415.431187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742058496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767415.431200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767415.431204:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767415.431207:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767415.431211:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3303 00000100:00080000:0.0:1775767415.431215:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767415.431219:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767415.431224:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767415.431235:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742058624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767415.431248:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767415.431260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767415.431264:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767415.431267:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767415.431273:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3303 00000100:00080000:0.0:1775767415.431278:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767415.431283:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767415.431289:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767415.431294:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767415/real 1775767415] req@ffff90c203639500 x1862023742058368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767415.431304:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742058752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767415.431313:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742058368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767415.431325:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767415.431328:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767415.431332:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742058368:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767415.431356:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058496:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767415.431371:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767415.431379:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767415/real 1775767415] req@ffff90c203638000 x1862023742058496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767415.431386:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742058496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767415.431407:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767415.431411:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767415.431415:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767415.431421:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767415.431442:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767415.431454:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767415/real 1775767415] req@ffff90c203639180 x1862023742058624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767415.431466:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742058624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767415.431474:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767415.431478:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767415.431481:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767415.431488:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058752:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767415.431539:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767415.431549:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767415/real 1775767415] req@ffff90c20363aa00 x1862023742058752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767415.431558:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742058752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767415.431569:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767415.431573:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767415.431577:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742058752:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767416.455014:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767417.479085:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767418.503052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767419.527104:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767420.551029:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767420.551038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767420.551043:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767420.551052:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3309 00000100:00080000:0.0:1775767420.551063:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767420.551074:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767420.551092:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767420.551111:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742058880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767420.551132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767420.551135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767420.551136:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767420.551139:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3309 00000100:00080000:0.0:1775767420.551141:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767420.551151:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767420.551161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767420.551177:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742059008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767420.551181:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742058880:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767420.551200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767420.551202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767420.551204:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767420.551207:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3309 00000100:00080000:0.0:1775767420.551210:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767420.551213:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767420.551216:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767420.551224:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742059136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767420.551231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767420.551234:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767420.551235:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767420.551237:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3309 00000100:00080000:0.0:1775767420.551241:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767420.551244:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767420.551247:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767420.551254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742059264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767420.551275:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767420.551325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767420/real 1775767420] req@ffff90c203638700 x1862023742058880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767420.551347:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742058880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767420.551359:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767420.551364:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767420.551369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742058880:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767420.551400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767420.551422:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767420.551434:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767420/real 1775767420] req@ffff90c20363aa00 x1862023742059008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767420.551446:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742059008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767420.551455:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767420.551458:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767420.551462:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767420.551481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767420.551503:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767420.551527:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767420/real 1775767420] req@ffff90c203639180 x1862023742059136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767420.551538:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742059136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767420.551543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767420.551546:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767420.551548:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767420.551553:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059264:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767420.551569:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767420.551576:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767420/real 1775767420] req@ffff90c203638000 x1862023742059264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767420.551583:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742059264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767420.551591:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767420.551592:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767420.551594:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059264:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767421.575081:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767422.599047:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767423.623032:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767424.647058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767425.671019:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767425.671030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767425.671036:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767425.671056:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3314 00000100:00080000:0.0:1775767425.671070:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767425.671078:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767425.671125:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742059392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775767425.671139:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767425.671152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767425.671156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767425.671157:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767425.671161:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3314 00000100:00080000:0.0:1775767425.671163:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767425.671167:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767425.671176:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767425.671187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742059520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767425.671197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767425.671202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767425.671204:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767425.671208:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3314 00000100:00100000:2.0:1775767425.671208:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742059392:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767425.671213:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767425.671221:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767425.671226:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767425.671232:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742059648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767425.671243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767425.671246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767425.671247:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767425.671250:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3314 00000100:00080000:0.0:1775767425.671253:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767425.671255:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767425.671258:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767425.671266:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742059776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767425.671336:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767425.671372:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767425.671396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059776:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767426.695075:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767427.719055:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767428.743066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775767428.743141:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775767428.743179:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775767428.743195:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767428.743232:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767428.743240:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767428.743247:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767428.743282:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767425/real 1775767428] req@ffff90c203638e00 x1862023742059520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767428.743306:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742059520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767428.743317:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767428.743321:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767428.743331:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767428.743338:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767425/real 1775767428] req@ffff90c20363a680 x1862023742059648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767428.743350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742059648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767428.743357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767428.743361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767428.743364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767428.743371:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767425/real 1775767428] req@ffff90c20363b100 x1862023742059776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767428.743381:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742059776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767428.743401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767428.743405:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767428.743408:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742059776:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767428.743415:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767425/real 1775767428] req@ffff90c203639500 x1862023742059392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767450 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767428.743425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742059392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767450 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767428.743432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767428.743435:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767428.743437:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742059392:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775767429.767058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767430.791039:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767430.791077:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767430.791087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767430.791092:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767430.791104:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3319 00000100:00080000:1.0:1775767430.791118:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767430.791125:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767430.791169:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742059904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767430.791215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767430.791219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767430.791221:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767430.791225:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3319 00000100:00080000:1.0:1775767430.791229:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767430.791234:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767430.791256:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767430.791277:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742060032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767430.791282:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742059904:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767430.791302:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767430.791307:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767430.791309:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767430.791314:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3319 00000100:00080000:1.0:1775767430.791320:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767430.791337:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767430.791343:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767430.791363:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742060160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767430.791365:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767430.791410:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767430.791414:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767430.791417:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767430.791424:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3319 00000100:00080000:1.0:1775767430.791430:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767430.791437:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767430/real 1775767430] req@ffff90c203496d80 x1862023742059904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767430.791441:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767430.791455:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767430.791480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742059904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767430.791493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1775767430.791495:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742060288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767430.791503:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767430.791525:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742059904:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767430.791565:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767430.791590:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767430.791608:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767430/real 1775767430] req@ffff90c203495880 x1862023742060032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767430.791620:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742060032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767430.791628:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767430.791631:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767430.791634:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767430.791640:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767430.791658:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767430.791667:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767430/real 1775767430] req@ffff90c203497100 x1862023742060160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767430.791678:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742060160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767430.791685:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767430.791688:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767430.791691:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767430.791697:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060288:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767430.791715:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767430.791725:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767430/real 1775767430] req@ffff90c203495c00 x1862023742060288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767455 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767430.791735:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742060288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767455 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767430.791745:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767430.791748:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767430.791751:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060288:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767431.815023:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767432.839034:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767433.863014:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767434.887065:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767435.910993:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767435.911009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767435.911017:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767435.911022:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767435.911034:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3324 00000100:00080000:0.0:1775767435.911048:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767435.911057:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767435.911106:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742060416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767435.911133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767435.911137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767435.911138:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767435.911141:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3324 00000100:00080000:0.0:1775767435.911145:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767435.911148:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767435.911158:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767435.911169:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742060544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767435.911174:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742060416:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767435.911184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767435.911186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767435.911187:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767435.911190:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3324 00000100:00080000:0.0:1775767435.911192:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767435.911194:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767435.911198:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767435.911205:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742060672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767435.911211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767435.911221:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767435.911222:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767435.911225:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3324 00000100:00080000:0.0:1775767435.911228:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767435.911231:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767435.911235:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767435.911243:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742060800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767435.911260:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767435.911299:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767435/real 1775767435] req@ffff90c20363bb80 x1862023742060416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767435.911319:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742060416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767435.911344:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767435.911350:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767435.911355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742060416:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767435.911382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060544:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767435.911401:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767435.911411:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767435/real 1775767435] req@ffff90c203638380 x1862023742060544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767435.911420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742060544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767435.911430:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767435.911433:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767435.911436:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767435.911443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767435.911463:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767435.911475:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767435/real 1775767435] req@ffff90c20363b480 x1862023742060672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767435.911487:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742060672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767435.911495:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767435.911498:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767435.911501:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767435.911507:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060800:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767435.911527:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767435.911539:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767435/real 1775767435] req@ffff90c203639f80 x1862023742060800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767435.911552:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742060800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767460 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767435.911563:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767435.911567:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767435.911569:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742060800:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767436.935028:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767437.959001:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767438.983039:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767440.007028:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767441.030983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767441.030994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767441.030999:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767441.031009:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3329 00010000:00010000:0.0:1775767441.031019:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767441.031021:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767441.031028:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767441.031066:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742060928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767441.031095:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767441.031099:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767441.031101:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767441.031106:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3329 00000100:00080000:1.0:1775767441.031110:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767441.031115:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767441.031124:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767441.031136:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742061056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767441.031146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767441.031149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767441.031150:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742060928:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767441.031151:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767441.031158:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3329 00000100:00080000:1.0:1775767441.031167:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767441.031171:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767441.031176:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767441.031190:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742061184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767441.031216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767441.031220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767441.031222:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767441.031227:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767441.031231:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3329 00000100:00080000:1.0:1775767441.031237:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767441.031241:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767441.031246:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767441.031257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742061312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767441.031285:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767441/real 1775767441] req@ffff90c203494000 x1862023742060928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767441.031317:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742060928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767441.031328:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767441.031332:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767441.031337:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742060928:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767441.031360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061056:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767441.031379:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767441.031391:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767441/real 1775767441] req@ffff90c203494e00 x1862023742061056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767441.031404:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742061056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767441.031413:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767441.031416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767441.031419:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767441.031424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061184:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767441.031440:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767441.031448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767441/real 1775767441] req@ffff90c203495f80 x1862023742061184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767441.031459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742061184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767441.031466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767441.031479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767441.031482:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767441.031488:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061312:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767441.031504:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767441.031513:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767441/real 1775767441] req@ffff90c203494380 x1862023742061312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767441.031523:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742061312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767441.031532:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767441.031535:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767441.031539:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061312:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767442.055057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767443.079053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767444.103043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767445.127090:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767446.151007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767446.151017:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767446.151022:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767446.151033:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3334 00000100:00080000:0.0:1775767446.151045:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767446.151053:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767446.151071:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767446.151093:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742061440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767446.151123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767446.151127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767446.151129:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767446.151135:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3334 00000100:00080000:0.0:1775767446.151139:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767446.151144:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767446.151153:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742061440:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775767446.151154:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767446.151176:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742061568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767446.151190:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767446.151194:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767446.151195:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767446.151200:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3334 00000100:00080000:0.0:1775767446.151204:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767446.151211:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767446.151212:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767446.151235:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767446.151251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742061696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767446.151263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767446.151264:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767446/real 1775767446] req@ffff90c20363b800 x1862023742061440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767446.151270:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767446.151275:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767446.151282:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3334 00000100:00080000:0.0:1775767446.151287:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767446.151287:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742061440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767446.151297:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775767446.151300:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1775767446.151305:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775767446.151305:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767446.151311:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742061440:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767446.151323:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742061824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767446.151331:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767446.151352:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767446.151360:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767446/real 1775767446] req@ffff90c203639f80 x1862023742061568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767446.151370:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742061568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767446.151376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767446.151378:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767446.151380:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767446.151383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767446.151392:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767446.151398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767446/real 1775767446] req@ffff90c20363b480 x1862023742061696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767446.151405:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742061696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767446.151409:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767446.151411:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767446.151413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767446.151421:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061824:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767446.151431:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767446.151437:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767446/real 1775767446] req@ffff90c203638380 x1862023742061824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767446.151443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742061824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767446.151453:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767446.151455:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767446.151457:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742061824:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767447.175063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767448.199100:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767449.223008:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767450.247053:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767451.270969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767451.270978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767451.270983:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767451.270992:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3339 00000100:00080000:0.0:1775767451.271003:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767451.271010:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767451.271060:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767451.271065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742061952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767451.271092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767451.271096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767451.271097:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767451.271100:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3339 00000100:00080000:0.0:1775767451.271120:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767451.271123:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767451.271131:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767451.271143:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742062080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767451.271144:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742061952:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767451.271157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767451.271160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767451.271161:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767451.271163:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3339 00000100:00080000:0.0:1775767451.271166:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767451.271168:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767451.271187:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767451.271196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742062208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767451.271207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767451.271210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767451.271211:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767451.271214:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3339 00000100:00080000:0.0:1775767451.271216:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767451.271218:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767451.271222:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767451.271233:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742062336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767451.271234:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767451.271277:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767451/real 1775767451] req@ffff90c20363bb80 x1862023742061952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767451.271290:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742061952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767451.271298:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767451.271301:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767451.271304:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742061952:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767451.271322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062080:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767451.271336:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767451.271343:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767451/real 1775767451] req@ffff90c203639500 x1862023742062080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767451.271351:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742062080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767451.271356:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767451.271358:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767451.271360:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767451.271365:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062208:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767451.271374:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767451.271380:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767451/real 1775767451] req@ffff90c20363b100 x1862023742062208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767451.271387:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742062208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767451.271391:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767451.271393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767451.271395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767451.271398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062336:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767451.271407:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767451.271412:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767451/real 1775767451] req@ffff90c20363a680 x1862023742062336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767451.271418:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742062336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767451.271424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767451.271426:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767451.271428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062336:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767452.295050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767453.318977:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767454.343016:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767455.366993:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767456.390978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767456.390985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767456.390989:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767456.390998:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3344 00000100:00080000:1.0:1775767456.391008:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767456.391014:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767456.391046:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767456.391049:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742062464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767456.391071:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767456.391074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767456.391075:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767456.391078:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3344 00000100:00080000:1.0:1775767456.391081:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767456.391083:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767456.391092:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767456.391103:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742062592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767456.391103:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742062464:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767456.391119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767456.391133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767456.391135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767456.391140:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3344 00000100:00080000:1.0:1775767456.391144:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767456.391149:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767456.391155:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767456.391168:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742062720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767456.391174:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767456.391184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767456.391186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767456.391187:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767456.391192:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3344 00000100:00080000:1.0:1775767456.391195:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767456.391198:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767456.391203:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767456.391209:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767456/real 1775767456] req@ffff90c203497480 x1862023742062464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767456.391215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742062848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767456.391234:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742062464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767456.391243:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767456.391247:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767456.391250:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742062464:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767456.391268:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062592:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767456.391281:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767456.391288:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767456/real 1775767456] req@ffff90c203494380 x1862023742062592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767456.391296:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742062592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767456.391302:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767456.391304:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767456.391306:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767456.391309:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767456.391318:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767456.391323:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767456/real 1775767456] req@ffff90c203495f80 x1862023742062720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767456.391330:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742062720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767456.391339:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767456.391341:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767456.391344:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767456.391347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062848:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767456.391356:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767456.391361:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767456/real 1775767456] req@ffff90c203494e00 x1862023742062848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767456.391367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742062848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767456.391373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767456.391375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767456.391378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742062848:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767457.415019:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767458.439089:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767459.463079:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767460.487136:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767461.511016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767461.511026:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767461.511031:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767461.511043:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3349 00000100:00080000:3.0:1775767461.511056:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767461.511065:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775767461.511108:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767461.511135:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742062976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767461.511161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767461.511164:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767461.511165:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767461.511169:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3349 00000100:00080000:3.0:1775767461.511172:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767461.511175:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767461.511185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767461.511207:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742063104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767461.511208:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742062976:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767461.511228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767461.511232:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767461.511234:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767461.511240:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3349 00000100:00080000:3.0:1775767461.511244:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767461.511248:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767461.511254:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767461.511265:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742063232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767461.511272:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767461.511274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767461.511275:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767461.511279:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3349 00000100:00080000:3.0:1775767461.511283:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767461.511285:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767461.511289:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767461.511295:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767461.511299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742063360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767461.511347:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767461/real 1775767461] req@ffff90c218da3100 x1862023742062976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767461.511367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742062976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767461.511379:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767461.511384:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767461.511389:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742062976:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767461.511420:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767461.511442:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767461.511454:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767461/real 1775767461] req@ffff90c218da0000 x1862023742063104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767461.511467:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742063104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767461.511477:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767461.511481:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767461.511485:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767461.511491:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767461.511510:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767461.511521:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767461/real 1775767461] req@ffff90c218da1f80 x1862023742063232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767461.511544:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742063232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767461.511554:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767461.511557:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767461.511561:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767461.511568:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063360:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767461.511588:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767461.511599:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767461/real 1775767461] req@ffff90c218da0a80 x1862023742063360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767461.511612:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742063360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767461.511623:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767461.511627:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767461.511630:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063360:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767462.535039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767463.559094:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767464.583093:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767465.607066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767466.631055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:1.0:1775767466.631070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767466.631071:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767466.631078:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767466.631093:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3355 00000100:00080000:0.0:1775767466.631108:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767466.631117:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767466.631165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742063488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767466.631194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767466.631197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767466.631199:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767466.631202:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3355 00000100:00080000:0.0:1775767466.631205:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767466.631208:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767466.631218:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767466.631226:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742063616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767466.631233:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767466.631236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767466.631237:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767466.631239:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3355 00000100:00080000:0.0:1775767466.631241:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767466.631243:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767466.631246:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767466.631251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742063744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767466.631258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767466.631260:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767466.631261:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767466.631265:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3355 00000100:00080000:0.0:1775767466.631267:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767466.631270:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767466.631281:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767466.631292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742063872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767466.631293:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742063488:192.168.202.155@tcp:250:kworker.0 00000800:00000100:2.0:1775767466.631384:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767466.631417:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767466/real 1775767466] req@ffff90c203638e00 x1862023742063488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767466.631432:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742063488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767466.631441:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767466.631444:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767466.631447:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742063488:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767466.631454:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063616:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767466.631468:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767466.631475:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767466/real 1775767466] req@ffff90c203638000 x1862023742063616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767466.631483:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742063616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767466.631493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767466.631496:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767466.631498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767466.631502:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063744:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767466.631512:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767466.631517:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767466/real 1775767466] req@ffff90c203639180 x1862023742063744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767466.631524:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742063744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767466.631529:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767466.631531:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767466.631533:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767466.631563:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063872:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767466.631575:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767466.631581:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767466/real 1775767466] req@ffff90c20363aa00 x1862023742063872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767491 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767466.631588:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742063872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767491 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767466.631596:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767466.631598:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767466.631600:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742063872:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767467.655058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767468.679027:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767469.703065:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767470.727101:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767471.751009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:0.0:1775767471.751017:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767471.751019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767471.751026:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767471.751037:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3360 00000100:00080000:3.0:1775767471.751048:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767471.751055:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767471.751092:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742064000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767471.751121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767471.751125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767471.751127:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767471.751132:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3360 00000100:00080000:3.0:1775767471.751136:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767471.751141:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767471.751150:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742064000:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775767471.751152:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767471.751168:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742064128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767471.751180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767471.751184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767471.751187:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767471.751191:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3360 00000100:00080000:3.0:1775767471.751197:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767471.751202:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767471.751207:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775767471.751210:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767471.751223:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742064256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767471.751231:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767471.751233:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767471.751235:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767471.751241:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767471/real 1775767471] req@ffff90c218da0380 x1862023742064000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767471.751241:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3360 00000100:00080000:3.0:1775767471.751246:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767471.751253:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767471.751259:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767471.751265:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742064000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767471.751270:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742064384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767471.751277:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767471.751285:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767471.751290:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742064000:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767471.751309:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767471.751324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767471.751331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767471/real 1775767471] req@ffff90c218da0700 x1862023742064128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767471.751339:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742064128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767471.751345:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767471.751347:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767471.751350:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767471.751353:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767471.751363:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767471.751369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767471/real 1775767471] req@ffff90c218da3480 x1862023742064256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767471.751378:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742064256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767471.751387:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767471.751390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767471.751394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767471.751400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064384:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767471.751420:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767471.751442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767471/real 1775767471] req@ffff90c218da0e00 x1862023742064384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767496 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767471.751455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742064384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767496 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767471.751462:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767471.751464:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767471.751466:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064384:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767472.775019:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767473.799124:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767474.823120:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767475.847160:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767476.871009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767476.871020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767476.871026:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767476.871039:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3365 00000100:00080000:3.0:1775767476.871057:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767476.871063:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775767476.871100:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767476.871100:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742064512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767476.871125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767476.871132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767476.871134:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767476.871137:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3365 00000100:00080000:3.0:1775767476.871140:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767476.871143:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767476.871152:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767476.871161:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742064640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767476.871171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767476.871179:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767476.871181:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742064512:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767476.871181:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767476.871198:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3365 00000100:00080000:3.0:1775767476.871201:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767476.871203:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767476.871206:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767476.871217:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742064768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767476.871228:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767476.871230:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767476.871232:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767476.871235:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3365 00000100:00080000:3.0:1775767476.871237:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767476.871239:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767476.871243:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767476.871251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742064896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767476.871278:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767476.871330:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767476/real 1775767476] req@ffff90c218da2d80 x1862023742064512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767476.871353:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742064512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767476.871367:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767476.871372:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767476.871376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742064512:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767476.871402:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767476.871423:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767476.871433:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767476/real 1775767476] req@ffff90c218da3800 x1862023742064640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767476.871441:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742064640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767476.871447:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767476.871450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767476.871452:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767476.871456:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767476.871467:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767476.871473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767476/real 1775767476] req@ffff90c218da1880 x1862023742064768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767476.871480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742064768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767476.871486:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767476.871488:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767476.871489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767476.871494:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064896:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767476.871505:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767476.871510:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767476/real 1775767476] req@ffff90c218da0e00 x1862023742064896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767476.871518:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742064896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767501 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767476.871525:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767476.871527:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767476.871529:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742064896:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767477.895160:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767478.919128:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767479.943050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767480.967066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767481.990996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767481.991003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767481.991007:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767481.991016:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3370 00000100:00080000:0.0:1775767481.991026:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767481.991032:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767481.991055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767481.991066:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742065024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767481.991091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767481.991094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767481.991095:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767481.991099:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3370 00000100:00080000:0.0:1775767481.991101:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767481.991103:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767481.991111:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767481.991121:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742065152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767481.991130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767481.991133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767481.991133:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742065024:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767481.991135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767481.991142:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3370 00000100:00080000:0.0:1775767481.991144:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767481.991146:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767481.991149:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767481.991159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742065280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767481.991165:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767481.991168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767481.991169:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767481.991172:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3370 00000100:00080000:0.0:1775767481.991174:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767481.991176:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767481.991179:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767481.991185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742065408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767481.991199:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767481.991239:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767481/real 1775767481] req@ffff90c203638700 x1862023742065024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767481.991256:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742065024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767481.991263:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767481.991266:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767481.991269:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742065024:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767481.991288:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065152:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767481.991309:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767481.991322:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767481/real 1775767481] req@ffff90c20363aa00 x1862023742065152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767481.991337:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742065152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767481.991346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767481.991349:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767481.991353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767481.991359:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767481.991375:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767481.991383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767481/real 1775767481] req@ffff90c203639180 x1862023742065280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767481.991391:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742065280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767481.991396:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767481.991398:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767481.991400:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767481.991404:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065408:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767481.991414:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767481.991419:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767481/real 1775767481] req@ffff90c203638000 x1862023742065408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767481.991426:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742065408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767506 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767481.991437:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767481.991439:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767481.991441:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065408:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767483.015036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767484.039036:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767485.063051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767486.087060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767487.111000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767487.111020:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767487.111029:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767487.111038:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3375 00000100:00080000:0.0:1775767487.111051:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767487.111057:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767487.111071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767487.111101:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742065536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767487.111138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767487.111143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767487.111145:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767487.111149:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3375 00000100:00080000:0.0:1775767487.111154:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767487.111159:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767487.111169:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767487.111181:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742065536:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767487.111187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742065664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767487.111215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767487.111219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767487.111221:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767487.111226:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3375 00000100:00080000:0.0:1775767487.111230:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767487.111234:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767487.111240:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767487.111253:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742065792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767487.111262:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767487.111263:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767487.111276:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767487.111279:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767487.111284:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3375 00000100:00080000:0.0:1775767487.111287:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767487.111290:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767487.111295:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767487.111306:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767487/real 1775767487] req@ffff90c203638e00 x1862023742065536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767487.111307:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742065920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767487.111352:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742065536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767487.111364:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767487.111369:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767487.111373:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742065536:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767487.111398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767487.111422:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767487.111434:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767487/real 1775767487] req@ffff90c20363a680 x1862023742065664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767487.111448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742065664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767487.111466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767487.111470:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767487.111474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767487.111481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767487.111503:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767487.111516:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767487/real 1775767487] req@ffff90c20363b100 x1862023742065792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767487.111529:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742065792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767487.111538:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767487.111542:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767487.111545:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767487.111551:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065920:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767487.111570:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767487.111582:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767487/real 1775767487] req@ffff90c203639500 x1862023742065920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767487.111594:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742065920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767487.111605:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767487.111609:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767487.111612:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742065920:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767488.135043:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767489.159018:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767490.183001:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767491.207056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767492.230981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767492.230992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767492.230996:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767492.231004:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3380 00000100:00080000:0.0:1775767492.231014:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767492.231019:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767492.231046:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767492.231054:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742066048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767492.231074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767492.231077:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767492.231078:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767492.231081:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3380 00000100:00080000:0.0:1775767492.231083:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767492.231085:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767492.231093:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767492.231105:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742066176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767492.231106:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742066048:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767492.231119:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767492.231121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767492.231122:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767492.231125:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3380 00000100:00080000:0.0:1775767492.231128:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767492.231130:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767492.231133:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767492.231142:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742066304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767492.231149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767492.231151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767492.231152:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767492.231154:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3380 00000100:00080000:0.0:1775767492.231157:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767492.231159:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767492.231162:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767492.231168:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742066432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767492.231200:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767492.231235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767492.231260:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066432:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767493.255042:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767494.279056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767495.303040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775767495.303184:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775767495.303244:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767495.303267:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767495.303312:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767495.303325:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767495.303335:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767495.303389:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767492/real 1775767495] req@ffff90c203638380 x1862023742066176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767495.303425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742066176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767495.303438:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767495.303443:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767495.303448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767495.303456:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767492/real 1775767495] req@ffff90c20363b480 x1862023742066304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767495.303467:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742066304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767495.303475:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767495.303479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767495.303482:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767495.303489:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767492/real 1775767495] req@ffff90c203639f80 x1862023742066432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767495.303501:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742066432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767495.303519:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767495.303523:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767495.303527:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066432:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767495.303535:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767492/real 1775767495] req@ffff90c20363bb80 x1862023742066048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767495.303548:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742066048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767495.303557:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767495.303561:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767495.303564:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742066048:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767496.326995:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767497.350966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767497.350975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767497.350979:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767497.350987:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3385 00000100:00080000:0.0:1775767497.350998:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767497.351004:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767497.351019:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767497.351037:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742066560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767497.351060:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767497.351062:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767497.351064:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767497.351067:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3385 00000100:00080000:0.0:1775767497.351069:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767497.351072:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767497.351080:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767497.351091:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742066688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767497.351095:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742066560:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767497.351108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767497.351111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767497.351112:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767497.351115:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3385 00000100:00080000:0.0:1775767497.351117:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767497.351119:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767497.351123:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767497.351130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742066816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767497.351137:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767497.351140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767497.351141:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767497.351144:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3385 00000100:00080000:0.0:1775767497.351147:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767497.351149:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767497.351152:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767497.351160:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742066944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767497.351166:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767497.351202:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767497/real 1775767497] req@ffff90c20363b800 x1862023742066560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767497.351215:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742066560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767497.351223:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767497.351226:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767497.351229:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742066560:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767497.351249:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767497.351266:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767497.351274:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767497/real 1775767497] req@ffff90c20363bb80 x1862023742066688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767497.351282:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742066688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767497.351299:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767497.351303:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767497.351306:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767497.351312:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767497.351332:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767497.351343:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767497/real 1775767497] req@ffff90c203639f80 x1862023742066816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767497.351356:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742066816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767497.351366:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767497.351369:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767497.351372:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767497.351377:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066944:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767497.351397:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767497.351409:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767497/real 1775767497] req@ffff90c20363b480 x1862023742066944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767497.351430:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742066944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767497.351441:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767497.351445:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767497.351448:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742066944:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767498.375080:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767499.399049:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767500.423077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767501.447065:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767502.470924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767502.470931:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767502.470936:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767502.470944:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3390 00000100:00080000:0.0:1775767502.470954:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767502.470961:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767502.471004:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742067072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767502.471033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767502.471036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767502.471038:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767502.471041:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3390 00000100:00080000:0.0:1775767502.471046:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767502.471049:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767502.471057:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767502.471069:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742067200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767502.471080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767502.471083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767502.471086:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775767502.471090:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767502.471091:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3390 00000100:00100000:1.0:1775767502.471091:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742067072:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767502.471101:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767502.471105:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767502.471111:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767502.471128:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742067328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767502.471135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767502.471138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767502.471139:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767502.471142:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3390 00000100:00080000:0.0:1775767502.471144:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767502.471147:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767502.471150:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:1.0:1775767502.471158:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767502.471160:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742067456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767502.471195:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767502/real 1775767502] req@ffff90c203638380 x1862023742067072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767502.471208:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742067072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767502.471216:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767502.471219:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767502.471221:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742067072:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767502.471238:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767502.471252:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767502.471259:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767502/real 1775767502] req@ffff90c203639500 x1862023742067200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767502.471267:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742067200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767502.471272:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767502.471275:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767502.471276:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767502.471280:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767502.471290:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767502.471295:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767502/real 1775767502] req@ffff90c20363b100 x1862023742067328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767502.471302:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742067328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767502.471307:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767502.471309:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767502.471310:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767502.471314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067456:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775767502.471323:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767502.471328:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767502/real 1775767502] req@ffff90c20363a680 x1862023742067456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767502.471334:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742067456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767502.471341:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767502.471343:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767502.471344:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067456:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767503.495087:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767504.519013:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767505.543010:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767506.567035:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767507.591025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767507.591036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767507.591042:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767507.591052:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3396 00000100:00080000:0.0:1775767507.591067:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767507.591074:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767507.591083:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767507.591120:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742067584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767507.591147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767507.591150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767507.591151:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767507.591154:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3396 00000100:00080000:0.0:1775767507.591158:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767507.591161:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767507.591171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767507.591185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742067712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767507.591188:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742067584:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767507.591210:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767507.591213:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767507.591214:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767507.591218:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3396 00000100:00080000:0.0:1775767507.591220:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767507.591224:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767507.591227:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767507.591238:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742067840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767507.591246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767507.591249:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767507.591250:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767507.591253:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3396 00000100:00080000:0.0:1775767507.591256:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767507.591259:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767507.591262:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:1.0:1775767507.591269:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767507.591273:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742067968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767507.591316:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767507/real 1775767507] req@ffff90c203638e00 x1862023742067584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767507.591338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742067584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767507.591350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767507.591355:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767507.591359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742067584:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767507.591382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067712:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767507.591401:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767507.591410:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767507/real 1775767507] req@ffff90c203638000 x1862023742067712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767507.591419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742067712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767507.591424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767507.591427:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767507.591429:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767507.591433:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067840:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767507.591444:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767507.591450:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767507/real 1775767507] req@ffff90c203639180 x1862023742067840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767507.591462:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742067840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767507.591467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767507.591469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767507.591471:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767507.591475:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067968:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775767507.591484:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767507.591489:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767507/real 1775767507] req@ffff90c20363aa00 x1862023742067968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767507.591496:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742067968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767507.591504:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767507.591508:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767507.591511:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742067968:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767508.615100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767509.639003:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767510.663138:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767511.686975:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767512.711008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767512.711018:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767512.711024:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767512.711034:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3401 00010000:00010000:1.0:1775767512.711096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767512.711097:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767512.712298:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767512.712347:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742068096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767512.712373:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767512.712378:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767512.712380:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767512.712384:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3401 00000100:00080000:0.0:1775767512.712390:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767512.712394:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767512.712406:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742068096:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775767512.712417:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767512.712437:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742068224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767512.712451:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767512.712455:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767512.712457:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1775767512.712462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767512.712463:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3401 00000100:00080000:0.0:1775767512.712470:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767512.712475:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767512.712481:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767512.712496:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742068352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767512.712500:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767512/real 1775767512] req@ffff90c203638700 x1862023742068096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767512.712533:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767512.712538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767512.712541:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1775767512.712544:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742068096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767512.712549:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3401 00000100:00080000:0.0:1775767512.712556:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767512.712559:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767512.712560:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:1.0:1775767512.712565:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000008:00100000:0.0:1775767512.712567:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767512.712571:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742068096:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767512.712597:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742068480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767512.712619:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767512.712645:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767512.712658:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767512/real 1775767512] req@ffff90c20363aa00 x1862023742068224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767512.712666:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742068224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767512.712672:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767512.712674:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767512.712677:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767512.712699:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767512.712712:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767512.712719:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767512/real 1775767512] req@ffff90c203639180 x1862023742068352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767512.712726:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742068352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767512.712735:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767512.712737:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767512.712739:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767512.712748:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068480:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775767512.712758:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767512.712764:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767512/real 1775767512] req@ffff90c203638000 x1862023742068480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767537 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767512.712787:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742068480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767537 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767512.712794:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767512.712796:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767512.712798:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068480:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767513.735123:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767514.759047:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767515.783152:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767516.807104:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767517.831007:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767517.831016:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767517.831036:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767517.831046:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3406 00000100:00080000:3.0:1775767517.831057:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767517.831064:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775767517.831100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767517.831107:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742068608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767517.831150:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767517.831153:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767517.831155:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767517.831158:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3406 00000100:00080000:3.0:1775767517.831160:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767517.831163:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767517.831172:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767517.831180:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742068736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767517.831187:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767517.831189:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767517.831190:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767517.831193:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3406 00000100:00080000:3.0:1775767517.831195:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767517.831197:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767517.831200:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767517.831208:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742068864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767517.831214:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767517.831233:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767517.831235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767517.831239:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3406 00000100:00080000:3.0:1775767517.831242:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1775767517.831247:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742068608:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767517.831248:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767517.831259:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767517.831271:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742068992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1775767517.831321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767517.831358:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767517/real 1775767517] req@ffff90c218da3480 x1862023742068608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767517.831375:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742068608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767517.831390:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767517.831394:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767517.831398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742068608:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767517.831403:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068736:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767517.831419:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767517.831429:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767517/real 1775767517] req@ffff90c218da0700 x1862023742068736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767517.831458:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742068736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767517.831467:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767517.831471:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767517.831474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767517.831479:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068864:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767517.831511:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767517.831523:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767517/real 1775767517] req@ffff90c218da0380 x1862023742068864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767517.831558:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742068864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767517.831564:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767517.831566:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767517.831568:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767517.831589:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068992:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775767517.831604:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767517.831611:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767517/real 1775767517] req@ffff90c218da0a80 x1862023742068992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767517.831618:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742068992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767542 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767517.831625:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767517.831627:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767517.831629:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742068992:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767518.855001:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767519.879126:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767520.903062:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767521.927045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767522.950996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767522.951007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:0.0:1775767522.951012:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767522.951013:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767522.951029:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3411 00000100:00080000:1.0:1775767522.951041:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767522.951048:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767522.951093:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742069120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767522.951122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767522.951125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767522.951127:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767522.951131:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3411 00000100:00080000:1.0:1775767522.951134:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767522.951138:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767522.951149:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767522.951162:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742069248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767522.951163:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742069120:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767522.951189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767522.951191:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767522.951192:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767522.951197:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3411 00000100:00080000:1.0:1775767522.951202:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767522.951206:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767522.951210:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767522.951218:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742069376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767522.951225:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767522.951228:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767522.951230:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767522.951234:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3411 00000100:00080000:1.0:1775767522.951237:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767522.951239:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767522.951243:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775767522.951244:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767522.951255:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742069504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767522.951286:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767522/real 1775767522] req@ffff90c203494000 x1862023742069120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767522.951301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742069120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767522.951310:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767522.951313:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767522.951316:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742069120:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767522.951336:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767522.951350:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767522.951357:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767522/real 1775767522] req@ffff90c203495c00 x1862023742069248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767522.951365:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742069248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767522.951375:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767522.951377:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767522.951379:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767522.951383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767522.951393:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767522.951398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767522/real 1775767522] req@ffff90c203497100 x1862023742069376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767522.951405:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742069376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767522.951410:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767522.951412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767522.951413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767522.951417:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069504:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767522.951426:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767522.951431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767522/real 1775767522] req@ffff90c203495880 x1862023742069504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767522.951440:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742069504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767547 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767522.951450:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767522.951454:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767522.951458:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069504:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767523.975040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767524.998997:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767526.023097:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767527.047013:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767528.070991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767528.070999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767528.071004:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767528.071013:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3416 00000100:00080000:1.0:1775767528.071026:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767528.071034:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767528.071073:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767528.071077:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742069632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767528.071107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767528.071112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767528.071114:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767528.071118:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3416 00000100:00080000:1.0:1775767528.071122:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767528.071126:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767528.071135:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767528.071160:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742069632:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767528.071169:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742069760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767528.071181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767528.071184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767528.071186:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767528.071189:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3416 00000100:00080000:1.0:1775767528.071193:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767528.071198:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767528.071202:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767528.071211:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742069888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767528.071219:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767528.071223:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767528.071244:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767528.071244:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767528.071252:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3416 00000100:00080000:1.0:1775767528.071256:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767528.071260:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767528.071264:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767528.071275:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742070016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767528.071288:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767528/real 1775767528] req@ffff90c203496d80 x1862023742069632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767528.071324:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742069632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767528.071333:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767528.071337:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767528.071341:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742069632:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767528.071364:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767528.071382:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767528.071392:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767528/real 1775767528] req@ffff90c203497b80 x1862023742069760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767528.071402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742069760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767528.071409:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767528.071412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767528.071446:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767528.071451:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767528.071467:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767528.071475:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767528/real 1775767528] req@ffff90c203496300 x1862023742069888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767528.071484:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742069888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767528.071489:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767528.071492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767528.071494:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742069888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767528.071498:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070016:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767528.071509:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767528.071514:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767528/real 1775767528] req@ffff90c203496a00 x1862023742070016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767528.071522:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742070016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767528.071529:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767528.071531:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767528.071533:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070016:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767529.095046:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767530.119084:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767531.143209:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767532.167045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767533.191021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767533.191032:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767533.191037:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767533.191050:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3421 00000100:00080000:1.0:1775767533.191063:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767533.191071:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767533.191092:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767533.191127:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742070144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767533.191152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767533.191155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767533.191157:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767533.191160:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3421 00000100:00080000:1.0:1775767533.191163:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767533.191166:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767533.191177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767533.191187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742070272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767533.191196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767533.191204:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767533.191205:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742070144:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767533.191206:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767533.191219:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3421 00000100:00080000:1.0:1775767533.191222:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767533.191224:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767533.191227:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767533.191237:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742070400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767533.191244:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767533.191247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767533.191249:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767533.191253:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3421 00000100:00080000:1.0:1775767533.191255:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767533.191258:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767533.191261:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767533.191268:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742070528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767533.191282:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767533.191328:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767533/real 1775767533] req@ffff90c203496680 x1862023742070144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767533.191344:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742070144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767533.191353:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767533.191356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767533.191359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742070144:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767533.191393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070272:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767533.191415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767533.191426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767533/real 1775767533] req@ffff90c203494a80 x1862023742070272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767533.191439:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742070272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767533.191447:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767533.191451:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767533.191454:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767533.191461:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070400:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767533.191482:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767533.191492:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767533/real 1775767533] req@ffff90c203494700 x1862023742070400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767533.191503:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742070400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767533.191511:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767533.191514:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767533.191517:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767533.191524:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070528:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767533.191544:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767533.191554:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767533/real 1775767533] req@ffff90c203495500 x1862023742070528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767533.191567:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742070528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767533.191577:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767533.191580:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767533.191582:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070528:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767534.215037:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767535.239058:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767536.263148:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767537.287048:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767538.310988:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767538.310998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767538.311004:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:1.0:1775767538.311014:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767538.311015:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3426 00000100:00080000:0.0:1775767538.311027:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767538.311034:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767538.311070:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742070656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767538.311093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767538.311095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767538.311097:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767538.311099:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3426 00000100:00080000:0.0:1775767538.311102:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767538.311106:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767538.311114:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767538.311125:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742070784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767538.311126:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742070656:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767538.311140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767538.311142:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767538.311144:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767538.311147:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3426 00000100:00080000:0.0:1775767538.311149:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767538.311151:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767538.311156:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767538.311164:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742070912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767538.311171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767538.311173:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767538.311175:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767538.311177:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3426 00000100:00080000:0.0:1775767538.311179:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767538.311181:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767538.311185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767538.311186:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767538.311195:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742071040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767538.311226:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767538/real 1775767538] req@ffff90c203638e00 x1862023742070656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767538.311247:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742070656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767538.311258:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767538.311262:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767538.311266:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742070656:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767538.311287:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767538.311305:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767538.311313:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767538/real 1775767538] req@ffff90c20363a680 x1862023742070784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767538.311323:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742070784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767538.311328:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767538.311356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767538.311358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767538.311363:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767538.311378:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767538.311386:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767538/real 1775767538] req@ffff90c20363b100 x1862023742070912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767538.311394:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742070912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767538.311399:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767538.311402:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767538.311403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742070912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767538.311407:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071040:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767538.311417:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767538.311422:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767538/real 1775767538] req@ffff90c203639500 x1862023742071040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767538.311429:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742071040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767538.311440:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767538.311442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767538.311444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071040:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767539.335041:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767540.359061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767541.383012:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767542.407017:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767543.430984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767543.430995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767543.431000:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767543.431010:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3431 00000100:00080000:1.0:1775767543.431021:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767543.431027:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767543.431058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767543.431062:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742071168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767543.431084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767543.431087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767543.431088:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767543.431091:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3431 00000100:00080000:1.0:1775767543.431094:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767543.431096:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767543.431103:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767543.431113:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742071296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767543.431121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767543.431125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767543.431127:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767543.431127:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742071168:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767543.431132:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3431 00000100:00080000:1.0:1775767543.431137:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767543.431140:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767543.431143:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767543.431150:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742071424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767543.431156:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767543.431159:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767543.431160:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767543.431163:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3431 00000100:00080000:1.0:1775767543.431165:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767543.431167:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767543.431170:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767543.431176:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742071552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767543.431192:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767543.431222:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767543/real 1775767543] req@ffff90c203495180 x1862023742071168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767543.431235:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742071168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767543.431243:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767543.431246:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767543.431249:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742071168:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767543.431267:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767543.431328:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767543.431342:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767543/real 1775767543] req@ffff90c203495500 x1862023742071296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767543.431355:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742071296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767543.431363:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767543.431366:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767543.431369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767543.431377:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071424:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767543.431397:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767543.431407:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767543/real 1775767543] req@ffff90c203494700 x1862023742071424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767543.431418:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742071424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767543.431426:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767543.431429:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767543.431432:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767543.431438:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071552:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767543.431457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767543.431467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767543/real 1775767543] req@ffff90c203494a80 x1862023742071552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767543.431478:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742071552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767543.431487:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767543.431491:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767543.431494:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071552:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767544.455026:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767545.479091:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767546.502985:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767547.527036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767548.550974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767548.550982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767548.550986:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767548.550994:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3437 00000100:00080000:0.0:1775767548.551003:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767548.551010:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767548.551055:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742071680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767548.551071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767548.551081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767548.551084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767548.551085:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767548.551088:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3437 00000100:00080000:0.0:1775767548.551092:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767548.551094:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767548.551103:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767548.551113:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742071808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767548.551121:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767548.551125:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1775767548.551125:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742071680:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767548.551127:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767548.551137:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3437 00000100:00080000:0.0:1775767548.551141:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767548.551145:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767548.551150:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767548.551165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742071936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767548.551176:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767548.551180:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767548.551182:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767548.551187:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3437 00000800:00000100:1.0:1775767548.551190:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767548.551195:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767548.551200:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767548.551205:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767548.551216:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742072064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767548.551227:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767548/real 1775767548] req@ffff90c203638380 x1862023742071680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767548.551267:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742071680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767548.551279:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767548.551282:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767548.551285:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742071680:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767548.551303:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071808:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767548.551320:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767548.551328:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767548/real 1775767548] req@ffff90c20363b480 x1862023742071808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767548.551336:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742071808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767548.551341:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767548.551343:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767548.551345:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767548.551349:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071936:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767548.551358:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767548.551363:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767548/real 1775767548] req@ffff90c203639f80 x1862023742071936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767548.551371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742071936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767548.551376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767548.551377:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767548.551379:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742071936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767548.551383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072064:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775767548.551406:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767548.551418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767548/real 1775767548] req@ffff90c20363bb80 x1862023742072064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767548.551429:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742072064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767548.551439:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767548.551443:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767548.551446:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072064:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767549.575135:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767550.599031:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767551.623056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767552.647079:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767553.671016:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767553.671029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767553.671034:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767553.671044:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3442 00000100:00080000:0.0:1775767553.671059:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767553.671068:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767553.671110:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767553.671124:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742072192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767553.671150:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767553.671154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767553.671155:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767553.671159:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3442 00000100:00080000:0.0:1775767553.671162:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767553.671166:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767553.671179:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767553.671204:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742072192:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767553.671228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742072320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767553.671241:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767553.671244:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767553.671246:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767553.671249:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3442 00000100:00080000:0.0:1775767553.671253:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767553.671257:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767553.671263:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767553.671272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742072448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767553.671280:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767553.671282:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767553.671283:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767553.671286:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3442 00000100:00080000:0.0:1775767553.671289:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1775767553.671291:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767553.671295:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767553.671299:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767553.671306:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742072576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767553.671345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767553/real 1775767553] req@ffff90c20363b800 x1862023742072192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767553.671365:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742072192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767553.671376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767553.671381:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767553.671386:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742072192:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767553.671413:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767553.671437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767553.671449:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767553/real 1775767553] req@ffff90c20363bb80 x1862023742072320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767553.671461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742072320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767553.671469:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767553.671472:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767553.671475:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767553.671480:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:1.0:1775767553.671499:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767553.671510:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767553/real 1775767553] req@ffff90c203639f80 x1862023742072448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767553.671521:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742072448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767553.671529:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767553.671533:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767553.671536:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767553.671542:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072576:192.168.202.155@tcp:8:kworker.0 00000800:00000100:1.0:1775767553.671560:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767553.671569:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767553/real 1775767553] req@ffff90c20363b480 x1862023742072576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767578 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767553.671588:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742072576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767578 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767553.671597:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775767553.671600:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:1.0:1775767553.671603:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072576:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767554.695075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767555.719095:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767556.743137:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767557.767100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767558.791009:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767558.791024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767558.791030:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767558.791043:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3447 00000100:00080000:0.0:1775767558.791067:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767558.791078:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767558.791114:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767558.791147:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742072704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767558.791185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767558.791189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767558.791192:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767558.791196:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3447 00000100:00080000:0.0:1775767558.791201:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767558.791206:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767558.791219:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767558.791236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742072832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767558.791240:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742072704:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767558.791257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767558.791261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767558.791263:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767558.791269:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3447 00000100:00080000:0.0:1775767558.791273:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767558.791277:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767558.791284:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767558.791296:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742072960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767558.791307:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767558.791311:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767558.791313:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767558.791318:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3447 00000100:00080000:0.0:1775767558.791322:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767558.791326:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767558.791332:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767558.791345:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742073088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1775767558.791369:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767558.791394:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:1.0:1775767558.791410:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073088:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767559.815185:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767560.839233:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767561.863356:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00010000:00010000:3.0:1775767561.863398:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767561.863420:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767561.863443:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767561.863497:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767561.863509:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767561.863521:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767561.863539:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767558/real 1775767561] req@ffff90c203639500 x1862023742072832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767561.863574:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742072832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767561.863592:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767561.863599:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767561.863603:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767561.863613:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767558/real 1775767561] req@ffff90c20363b100 x1862023742072960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767561.863628:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742072960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767561.863637:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767561.863642:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767561.863646:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742072960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767561.863654:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767558/real 1775767561] req@ffff90c20363a680 x1862023742073088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767561.863984:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742073088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767561.863997:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767561.864003:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767561.864006:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073088:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767561.864014:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767558/real 1775767561] req@ffff90c203638380 x1862023742072704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767583 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767561.864030:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742072704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767583 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767561.864040:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767561.864044:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767561.864047:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742072704:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775767562.887210:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767563.911036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767563.911048:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767563.911053:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767563.911065:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3452 00010000:00010000:3.0:1775767563.911069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767563.911079:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767563.911102:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767563.911141:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742073216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767563.911167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767563.911171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767563.911173:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767563.911177:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3452 00000100:00080000:0.0:1775767563.911181:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767563.911185:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767563.911196:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767563.911212:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742073344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767563.911226:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767563.911232:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767563.911235:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767563.911237:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742073216:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767563.911244:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3452 00000100:00080000:0.0:1775767563.911266:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767563.911273:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767563.911280:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767563.911296:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742073472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767563.911308:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767563.911312:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767563.911314:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767563.911318:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3452 00000100:00080000:0.0:1775767563.911322:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767563.911327:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767563.911334:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767563.911352:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767563.911356:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742073600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767563.911442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767563/real 1775767563] req@ffff90c203638e00 x1862023742073216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767563.911470:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742073216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767563.911484:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767563.911489:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767563.911494:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742073216:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767563.911537:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767563.911568:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767563.911583:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767563/real 1775767563] req@ffff90c203638000 x1862023742073344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767563.911603:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742073344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767563.911613:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767563.911617:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767563.911620:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767563.911627:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767563.911650:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767563.911661:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767563/real 1775767563] req@ffff90c203639180 x1862023742073472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767563.911674:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742073472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767563.911681:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767563.911685:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767563.911689:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767563.911696:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073600:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767563.911718:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767563.911729:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767563/real 1775767563] req@ffff90c20363aa00 x1862023742073600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767563.911741:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742073600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767588 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767563.911754:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767563.911757:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767563.911761:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073600:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767564.935257:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767565.959003:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767566.983022:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767568.007004:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767569.030996:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767569.031007:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767569.031014:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767569.031024:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3457 00010000:00010000:0.0:1775767569.031045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767569.031050:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767569.031060:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767569.031098:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742073728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767569.031121:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767569.031124:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767569.031125:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767569.031128:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3457 00000100:00080000:1.0:1775767569.031131:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767569.031134:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767569.031143:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767569.031152:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742073856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767569.031162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767569.031162:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742073728:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767569.031169:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767569.031172:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767569.031175:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3457 00000100:00080000:1.0:1775767569.031177:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767569.031179:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767569.031182:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767569.031190:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742073984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767569.031207:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767569.031210:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767569.031211:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767569.031214:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3457 00000100:00080000:1.0:1775767569.031217:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767569.031219:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767569.031222:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767569.031229:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767569.031232:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742074112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767569.031270:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767569/real 1775767569] req@ffff90c203496680 x1862023742073728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767569.031289:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742073728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767569.031300:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767569.031304:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767569.031309:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742073728:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767569.031331:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767569.031352:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767569.031364:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767569/real 1775767569] req@ffff90c203496a00 x1862023742073856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767569.031376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742073856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767569.031385:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767569.031388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767569.031391:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767569.031397:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767569.031414:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767569.031423:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767569/real 1775767569] req@ffff90c203496300 x1862023742073984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767569.031435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742073984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767569.031449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767569.031452:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767569.031455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742073984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767569.031471:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074112:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767569.031492:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767569.031501:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767569/real 1775767569] req@ffff90c203497b80 x1862023742074112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767569.031513:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742074112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767569.031523:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767569.031526:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767569.031529:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074112:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767570.054991:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767571.079033:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767572.103072:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767573.127025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767574.151003:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767574.151014:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767574.151019:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767574.151029:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3462 00000100:00080000:1.0:1775767574.151042:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767574.151050:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767574.151067:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767574.151091:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742074240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767574.151119:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767574.151123:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767574.151125:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767574.151130:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3462 00000100:00080000:1.0:1775767574.151136:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767574.151140:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767574.151147:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742074240:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775767574.151151:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767574.151168:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742074368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767574.151179:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767574.151182:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767574.151183:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767574.151186:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3462 00000100:00080000:1.0:1775767574.151189:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767574.151192:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767574.151197:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767574.151212:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767574.151214:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742074496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767574.151226:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767574.151230:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767574.151232:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767574.151237:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3462 00000100:00080000:1.0:1775767574.151242:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767574.151247:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767574/real 1775767574] req@ffff90c203496d80 x1862023742074240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767574.151248:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767574.151257:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767574.151267:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742074240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767574.151270:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742074624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767574.151294:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767574.151299:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767574.151303:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742074240:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767574.151325:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074368:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767574.151339:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767574.151347:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767574/real 1775767574] req@ffff90c203495880 x1862023742074368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767574.151355:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742074368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767574.151360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767574.151362:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767574.151364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767574.151368:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074496:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767574.151378:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767574.151383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767574/real 1775767574] req@ffff90c203497100 x1862023742074496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767574.151389:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742074496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767574.151394:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767574.151396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767574.151398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767574.151401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074624:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767574.151411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767574.151416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767574/real 1775767574] req@ffff90c203495c00 x1862023742074624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767574.151423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742074624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767574.151429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767574.151431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767574.151432:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074624:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767575.175057:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767576.199117:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767577.223085:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767578.247089:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767579.270990:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767579.271000:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767579.271004:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767579.271017:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3467 00000100:00080000:1.0:1775767579.271028:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767579.271036:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767579.271046:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767579.271073:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742074752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767579.271097:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767579.271101:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767579.271103:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767579.271107:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3467 00000100:00080000:1.0:1775767579.271111:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767579.271116:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767579.271125:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767579.271132:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742074752:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767579.271139:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742074880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767579.271151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767579.271154:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767579.271156:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767579.271160:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3467 00000100:00080000:1.0:1775767579.271164:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767579.271168:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767579.271173:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767579.271184:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742075008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767579.271193:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767579.271199:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767579.271200:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767579.271204:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767579.271209:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3467 00000100:00080000:1.0:1775767579.271214:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767579.271218:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767579.271224:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767579.271235:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742075136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767579.271246:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767579/real 1775767579] req@ffff90c203494000 x1862023742074752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767579.271268:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742074752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767579.271278:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767579.271283:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767579.271288:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742074752:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767579.271311:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767579.271329:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767579.271337:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767579/real 1775767579] req@ffff90c203494e00 x1862023742074880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767579.271346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742074880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767579.271357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767579.271359:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767579.271362:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742074880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767579.271366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767579.271376:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767579.271385:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767579/real 1775767579] req@ffff90c203495f80 x1862023742075008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767579.271397:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742075008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767579.271405:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767579.271408:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767579.271412:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767579.271419:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075136:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767579.271438:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767579.271449:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767579/real 1775767579] req@ffff90c203494380 x1862023742075136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767579.271459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742075136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767579.271466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767579.271468:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767579.271471:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075136:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767580.295065:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767581.319038:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767582.343023:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767583.367007:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767584.391003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767584.391014:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:1.0:1775767584.391017:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767584.391020:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767584.391028:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3472 00000100:00080000:0.0:1775767584.391038:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767584.391043:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767584.391075:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742075264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767584.391097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767584.391100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767584.391102:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767584.391105:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3472 00000100:00080000:0.0:1775767584.391108:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767584.391111:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767584.391119:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767584.391124:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742075264:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767584.391132:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742075392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767584.391140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767584.391143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767584.391143:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767584.391146:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3472 00000100:00080000:0.0:1775767584.391149:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767584.391151:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767584.391154:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767584.391161:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742075520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767584.391167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767584.391169:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767584.391170:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767584.391172:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3472 00000100:00080000:0.0:1775767584.391175:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767584.391177:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767584.391180:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767584.391185:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767584.391189:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742075648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767584.391220:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767584/real 1775767584] req@ffff90c203638700 x1862023742075264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767584.391233:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742075264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767584.391240:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767584.391243:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767584.391246:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742075264:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767584.391263:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075392:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767584.391276:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767584.391284:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767584/real 1775767584] req@ffff90c20363aa00 x1862023742075392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767584.391291:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742075392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767584.391296:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767584.391299:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767584.391301:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767584.391304:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075520:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767584.391313:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767584.391319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767584/real 1775767584] req@ffff90c203639180 x1862023742075520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767584.391326:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742075520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767584.391331:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767584.391332:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767584.391334:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767584.391338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075648:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767584.391354:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767584.391363:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767584/real 1775767584] req@ffff90c203638000 x1862023742075648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767584.391374:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742075648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767584.391383:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767584.391385:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767584.391388:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075648:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767585.415018:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767586.439123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767587.463001:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767588.487071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767589.510981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:3.0:1775767589.510991:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767589.510993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767589.510998:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767589.511015:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3477 00000100:00080000:0.0:1775767589.511025:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767589.511031:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767589.511065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742075776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767589.511086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767589.511089:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767589.511090:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767589.511093:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3477 00000100:00080000:0.0:1775767589.511096:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767589.511098:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767589.511106:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767589.511117:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742075904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767589.511117:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742075776:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767589.511131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767589.511134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767589.511135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767589.511143:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3477 00000100:00080000:0.0:1775767589.511145:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767589.511148:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767589.511153:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767589.511161:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742076032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767589.511168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767589.511170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767589.511172:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767589.511174:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3477 00000100:00080000:0.0:1775767589.511176:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767589.511178:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767589.511182:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767589.511185:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767589.511192:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742076160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767589.511231:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767589/real 1775767589] req@ffff90c203638e00 x1862023742075776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767589.511247:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742075776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767589.511256:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767589.511259:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767589.511262:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742075776:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767589.511281:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075904:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767589.511299:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767589.511308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767589/real 1775767589] req@ffff90c203638380 x1862023742075904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767589.511316:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742075904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767589.511321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767589.511324:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767589.511326:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742075904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767589.511329:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767589.511339:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767589.511345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767589/real 1775767589] req@ffff90c20363a680 x1862023742076032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767589.511352:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742076032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767589.511358:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767589.511359:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767589.511361:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767589.511366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076160:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767589.511375:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767589.511380:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767589/real 1775767589] req@ffff90c20363b100 x1862023742076160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767589.511387:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742076160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767589.511393:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767589.511395:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767589.511397:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076160:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767590.535121:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767591.559118:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767592.583081:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767593.607045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767594.630998:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767594.631006:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767594.631011:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767594.631021:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3483 00010000:00010000:0.0:1775767594.631025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767594.631031:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767594.631038:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767594.631072:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742076288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767594.631094:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767594.631096:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767594.631098:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767594.631101:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3483 00000100:00080000:3.0:1775767594.631104:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767594.631106:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767594.631114:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767594.631125:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742076416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767594.631129:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742076288:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767594.631142:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767594.631145:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767594.631146:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767594.631149:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3483 00000100:00080000:3.0:1775767594.631152:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767594.631155:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767594.631158:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767594.631168:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742076544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767594.631179:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767594.631182:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767594.631184:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767594.631186:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3483 00000100:00080000:3.0:1775767594.631189:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767594.631193:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767594.631194:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775767594.631198:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767594.631218:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742076672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767594.631240:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767594/real 1775767594] req@ffff90c218da1f80 x1862023742076288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767594.631260:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742076288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767594.631269:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767594.631272:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767594.631275:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742076288:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767594.631297:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767594.631314:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767594.631323:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767594/real 1775767594] req@ffff90c218da0000 x1862023742076416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767594.631332:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742076416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767594.631338:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767594.631340:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767594.631343:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076416:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767594.631346:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076544:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767594.631355:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767594.631361:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767594/real 1775767594] req@ffff90c218da3100 x1862023742076544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767594.631368:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742076544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767594.631373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767594.631375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767594.631377:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767594.631380:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076672:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767594.631390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767594.631395:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767594/real 1775767594] req@ffff90c218da0a80 x1862023742076672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767619 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767594.631403:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742076672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767619 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767594.631414:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767594.631416:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767594.631417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076672:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767595.655120:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767596.679047:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767597.703042:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767598.727042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767599.750997:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767599.751005:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767599.751010:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767599.751019:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3488 00000100:00080000:0.0:1775767599.751028:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767599.751034:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767599.751044:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767599.751071:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742076800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767599.751098:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767599.751103:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767599.751106:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767599.751111:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3488 00000100:00080000:0.0:1775767599.751116:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767599.751120:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767599.751128:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742076800:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775767599.751130:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767599.751146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742076928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767599.751155:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767599.751157:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767599.751159:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767599.751162:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3488 00000100:00080000:0.0:1775767599.751165:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767599.751168:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767599.751171:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767599.751178:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742077056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767599.751185:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767599.751187:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767599.751188:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767599.751190:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3488 00000100:00080000:0.0:1775767599.751193:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767599.751197:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767599.751199:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767599.751205:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767599.751213:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742077184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767599.751236:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767599/real 1775767599] req@ffff90c203639500 x1862023742076800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767599.751250:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742076800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767599.751258:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767599.751261:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767599.751264:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742076800:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767599.751281:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076928:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767599.751296:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767599.751303:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767599/real 1775767599] req@ffff90c20363b480 x1862023742076928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767599.751311:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742076928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767599.751317:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767599.751319:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767599.751321:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742076928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767599.751325:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077056:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767599.751334:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767599.751340:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767599/real 1775767599] req@ffff90c203639f80 x1862023742077056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767599.751347:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742077056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767599.751352:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767599.751353:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767599.751355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767599.751358:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077184:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767599.751367:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767599.751372:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767599/real 1775767599] req@ffff90c20363bb80 x1862023742077184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767624 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767599.751378:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742077184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767624 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767599.751384:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767599.751386:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767599.751388:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077184:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767600.775041:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767601.799014:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767602.823033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767603.847063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767604.870995:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767604.871005:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767604.871010:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:1.0:1775767604.871021:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767604.871022:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3493 00000100:00080000:3.0:1775767604.871035:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767604.871043:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767604.871079:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742077312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767604.871106:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767604.871111:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767604.871113:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767604.871118:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3493 00000100:00080000:3.0:1775767604.871122:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767604.871126:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767604.871137:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767604.871153:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742077440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767604.871154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742077312:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767604.871174:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767604.871178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767604.871181:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767604.871186:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3493 00000100:00080000:3.0:1775767604.871191:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767604.871196:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767604.871202:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767604.871215:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742077568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767604.871226:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767604.871227:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767604.871233:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767604.871235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767604.871242:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3493 00000100:00080000:3.0:1775767604.871247:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767604.871252:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767604.871259:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767604.871277:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767604/real 1775767604] req@ffff90c218da0380 x1862023742077312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767604.871277:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742077696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767604.871322:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742077312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767604.871333:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767604.871338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767604.871342:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742077312:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767604.871367:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767604.871389:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767604.871402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767604/real 1775767604] req@ffff90c218da0700 x1862023742077440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767604.871416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742077440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767604.871424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767604.871427:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767604.871431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767604.871437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767604.871457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767604.871468:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767604/real 1775767604] req@ffff90c218da3480 x1862023742077568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767604.871480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742077568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767604.871494:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767604.871498:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767604.871501:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767604.871508:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077696:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767604.871527:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767604.871537:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767604/real 1775767604] req@ffff90c218da0e00 x1862023742077696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767604.871549:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742077696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767629 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767604.871558:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767604.871572:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767604.871575:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077696:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767605.895051:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767606.919021:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767607.943054:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767608.967002:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767609.991010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767609.991021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767609.991026:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767609.991038:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3498 00000100:00080000:1.0:1775767609.991051:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767609.991065:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775767609.991085:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767609.991114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742077824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767609.991141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767609.991145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767609.991147:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767609.991150:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3498 00000100:00080000:1.0:1775767609.991155:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767609.991159:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767609.991168:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767609.991174:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742077824:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767609.991185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742077952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767609.991198:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767609.991202:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767609.991204:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767609.991208:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3498 00000100:00080000:1.0:1775767609.991212:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767609.991216:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767609.991221:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767609.991227:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767609.991233:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742078080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767609.991243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767609.991247:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767609.991248:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767609.991253:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3498 00000100:00080000:1.0:1775767609.991257:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767609.991260:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767609/real 1775767609] req@ffff90c203497480 x1862023742077824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767609.991264:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767609.991273:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767609.991283:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742077824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767609.991287:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742078208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767609.991296:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767609.991305:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767609.991309:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742077824:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767609.991330:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767609.991344:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767609.991352:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767609/real 1775767609] req@ffff90c203494380 x1862023742077952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767609.991360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742077952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767609.991365:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767609.991368:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767609.991370:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742077952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767609.991374:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078080:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767609.991390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767609.991400:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767609/real 1775767609] req@ffff90c203495f80 x1862023742078080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767609.991412:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742078080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767609.991421:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767609.991425:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767609.991428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767609.991434:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078208:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767609.991454:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767609.991466:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767609/real 1775767609] req@ffff90c203494e00 x1862023742078208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767609.991480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742078208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767634 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767609.991491:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767609.991495:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767609.991499:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078208:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767611.015064:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767612.039135:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767613.063057:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767614.087088:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767615.111034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767615.111046:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767615.111051:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767615.111062:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3503 00000100:00080000:1.0:1775767615.111075:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767615.111083:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767615.111103:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767615.111125:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742078336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767615.111154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767615.111158:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767615.111160:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767615.111164:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3503 00000100:00080000:1.0:1775767615.111169:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767615.111174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767615.111185:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767615.111197:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742078464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767615.111205:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742078336:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767615.111216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767615.111220:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767615.111222:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767615.111227:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3503 00000100:00080000:1.0:1775767615.111240:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767615.111244:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767615.111250:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767615.111264:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742078592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767615.111273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767615.111277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767615.111278:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767615.111282:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3503 00000100:00080000:1.0:1775767615.111288:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767615.111292:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767615.111298:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767615.111304:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767615.111316:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742078720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767615.111362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767615/real 1775767615] req@ffff90c203494000 x1862023742078336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767615.111383:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742078336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767615.111396:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767615.111401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767615.111406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742078336:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767615.111435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078464:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767615.111458:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767615.111470:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767615/real 1775767615] req@ffff90c203495c00 x1862023742078464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767615.111491:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742078464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767615.111506:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767615.111510:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767615.111513:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767615.111519:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078592:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767615.111538:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767615.111549:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767615/real 1775767615] req@ffff90c203497100 x1862023742078592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767615.111561:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742078592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767615.111568:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767615.111582:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767615.111616:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767615.111624:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078720:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767615.111646:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767615.111658:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767615/real 1775767615] req@ffff90c203495880 x1862023742078720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767615.111670:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742078720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767615.111681:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767615.111684:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767615.111688:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078720:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767616.135073:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767617.159155:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767618.183060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767619.207033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767620.231047:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767620.231061:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767620.231067:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:1.0:1775767620.231070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767620.231082:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3508 00000100:00080000:3.0:1775767620.231094:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767620.231101:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767620.231141:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742078848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767620.231178:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767620.231183:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767620.231186:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767620.231193:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3508 00000100:00080000:3.0:1775767620.231199:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767620.231204:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767620.231217:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767620.231231:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742078848:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767620.231236:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742078976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767620.231268:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767620.231274:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767620.231278:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767620.231285:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3508 00000100:00080000:3.0:1775767620.231290:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767620.231295:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767620.231303:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767620.231318:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742079104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767620.231326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767620.231335:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767620.231340:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767620.231343:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767620.231349:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3508 00000100:00080000:3.0:1775767620.231354:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767620.231360:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767620.231367:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767620.231377:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767620/real 1775767620] req@ffff90c218da1880 x1862023742078848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767620.231385:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742079232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767620.231407:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742078848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767620.231424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767620.231432:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767620.231439:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742078848:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767620.231477:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767620.231520:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767620.231595:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767620/real 1775767620] req@ffff90c218da3800 x1862023742078976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767620.231616:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742078976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767620.231626:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767620.231631:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767620.231635:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742078976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767620.231643:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767620.231671:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767620.231684:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767620/real 1775767620] req@ffff90c218da2d80 x1862023742079104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767620.231698:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742079104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767620.231706:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767620.231710:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767620.231713:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767620.231719:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079232:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767620.231737:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767620.231746:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767620/real 1775767620] req@ffff90c218da0e00 x1862023742079232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767620.231759:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742079232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767620.231769:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767620.231772:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767620.231775:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079232:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767621.255034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767622.279022:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767623.303016:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767624.327003:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767625.351000:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767625.351014:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767625.351020:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767625.351035:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3513 00000100:00080000:3.0:1775767625.351052:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767625.351061:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775767625.351072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767625.351117:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742079360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767625.351154:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767625.351158:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767625.351160:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767625.351164:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3513 00000100:00080000:3.0:1775767625.351168:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767625.351174:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767625.351186:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767625.351201:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742079488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767625.351211:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742079360:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767625.351219:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767625.351224:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767625.351226:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767625.351231:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3513 00000100:00080000:3.0:1775767625.351236:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767625.351242:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767625.351250:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767625.351266:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742079616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767625.351276:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767625.351281:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767625.351283:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767625.351288:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3513 00000100:00080000:3.0:1775767625.351293:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767625.351298:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767625.351307:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767625.351323:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:3.0:1775767625.351327:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742079744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767625.351358:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767625.351379:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079744:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767626.375014:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767627.399054:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767628.423095:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775767628.423196:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775767628.423240:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767628.423258:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767628.423304:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767628.423314:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767628.423323:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767628.423397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767625/real 1775767628] req@ffff90c218da0a80 x1862023742079744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767628.423426:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742079744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767628.423444:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767628.423450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767628.423456:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079744:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767628.423479:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767625/real 1775767628] req@ffff90c218da0700 x1862023742079488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767628.423492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742079488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767628.423502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767628.423505:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767628.423509:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767628.423516:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767625/real 1775767628] req@ffff90c218da0380 x1862023742079616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767628.423528:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742079616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767628.423537:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767628.423540:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767628.423543:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742079616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767628.423550:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767625/real 1775767628] req@ffff90c218da3480 x1862023742079360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767628.423564:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742079360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767628.423573:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767628.423576:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767628.423579:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742079360:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775767629.447122:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767630.471038:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767630.471049:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767630.471055:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767630.471066:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3518 00000100:00080000:3.0:1775767630.471092:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767630.471102:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767630.471145:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767630.471145:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742079872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767630.471183:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767630.471189:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767630.471194:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767630.471204:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3518 00000100:00080000:3.0:1775767630.471210:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767630.471226:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767630.471237:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767630.471243:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742079872:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767630.471260:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742080000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767630.471275:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767630.471279:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767630.471281:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767630.471289:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3518 00000100:00080000:3.0:1775767630.471295:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767630.471301:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767630.471308:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767630.471323:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742080128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767630.471334:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775767630.471336:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767630.471343:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767630.471345:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767630.471355:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3518 00000100:00080000:3.0:1775767630.471362:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767630.471370:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767630.471378:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767630.471404:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742080256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767630.471424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767630/real 1775767630] req@ffff90c218da3100 x1862023742079872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767630.471480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742079872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767630.471499:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767630.471507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767630.471514:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742079872:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767630.471553:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767630.471588:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767630.471606:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767630/real 1775767630] req@ffff90c218da0000 x1862023742080000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767630.471622:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742080000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767630.471633:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767630.471639:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767630.471644:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767630.471654:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767630.471681:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767630.471710:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767630/real 1775767630] req@ffff90c218da1f80 x1862023742080128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767630.471725:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742080128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767630.471736:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767630.471740:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767630.471744:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767630.471752:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080256:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767630.471778:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767630.471792:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767630/real 1775767630] req@ffff90c218da3480 x1862023742080256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767630.471807:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742080256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767630.471821:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767630.471825:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767630.471831:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080256:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775767631.495179:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767632.519078:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767633.543056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767634.566986:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767635.590960:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767635.590968:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767635.590973:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767635.590981:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3524 00000100:00080000:3.0:1775767635.590992:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767635.590999:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767635.591028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767635.591038:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742080384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767635.591079:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767635.591084:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767635.591087:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767635.591090:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3524 00000100:00080000:3.0:1775767635.591094:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767635.591096:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767635.591105:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767635.591116:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742080512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767635.591125:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767635.591127:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767635.591129:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775767635.591135:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742080384:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767635.591135:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3524 00000100:00080000:3.0:1775767635.591139:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767635.591145:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767635.591149:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767635.591157:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742080640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767635.591162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767635.591165:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767635.591167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767635.591170:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3524 00000100:00080000:3.0:1775767635.591172:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767635.591174:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767635.591177:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767635.591185:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742080768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767635.591205:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767635.591235:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767635/real 1775767635] req@ffff90c218da0380 x1862023742080384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767635.591249:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742080384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767635.591265:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767635.591268:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767635.591271:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742080384:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767635.591290:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767635.591305:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767635.591314:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767635/real 1775767635] req@ffff90c218da0700 x1862023742080512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767635.591322:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742080512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767635.591327:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767635.591329:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767635.591331:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767635.591335:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767635.591348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767635.591358:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767635/real 1775767635] req@ffff90c218da0a80 x1862023742080640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767635.591371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742080640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767635.591387:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767635.591402:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767635.591406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767635.591413:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080768:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767635.591433:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767635.591441:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767635/real 1775767635] req@ffff90c218da0e00 x1862023742080768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767635.591449:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742080768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767635.591456:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767635.591458:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767635.591460:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742080768:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767636.615082:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775767637.639022:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767638.663074:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767639.687004:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767640.710985:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767640.710995:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767640.711000:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767640.711011:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3529 00000100:00080000:0.0:1775767640.711024:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775767640.711025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767640.711031:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767640.711070:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742080896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767640.711093:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767640.711096:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767640.711098:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767640.711100:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3529 00000100:00080000:0.0:1775767640.711104:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767640.711107:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767640.711116:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767640.711125:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742080896:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767640.711129:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742081024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767640.711138:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767640.711141:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767640.711142:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767640.711145:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3529 00000100:00080000:0.0:1775767640.711147:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767640.711149:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767640.711152:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767640.711159:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742081152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767640.711165:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767640.711168:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767640.711170:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767640.711173:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3529 00000100:00080000:0.0:1775767640.711175:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767640.711177:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767640.711181:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767640.711181:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767640.711191:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742081280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767640.711215:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767640/real 1775767640] req@ffff90c20363b800 x1862023742080896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767640.711230:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742080896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767640.711238:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767640.711241:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767640.711244:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742080896:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767640.711260:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081024:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767640.711273:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767640.711291:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767640/real 1775767640] req@ffff90c20363bb80 x1862023742081024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767640.711299:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742081024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767640.711305:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767640.711307:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767640.711310:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767640.711313:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081152:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767640.711328:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767640.711333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767640/real 1775767640] req@ffff90c203639f80 x1862023742081152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767640.711363:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742081152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767640.711369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767640.711371:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767640.711373:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767640.711376:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081280:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767640.711390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767640.711397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767640/real 1775767640] req@ffff90c20363b480 x1862023742081280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767665 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767640.711404:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742081280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767665 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767640.711410:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767640.711412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767640.711415:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081280:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767641.735020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767642.759035:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767643.783001:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767644.807021:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767645.830942:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767645.830950:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767645.830955:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767645.830963:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3534 00000100:00080000:0.0:1775767645.830973:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767645.830979:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767645.830991:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767645.831014:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742081408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767645.831034:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767645.831037:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767645.831038:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767645.831041:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3534 00000100:00080000:0.0:1775767645.831043:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767645.831046:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767645.831054:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767645.831066:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742081536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767645.831066:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742081408:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767645.831080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767645.831082:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767645.831083:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767645.831087:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3534 00000100:00080000:0.0:1775767645.831089:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767645.831092:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767645.831095:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767645.831102:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742081664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767645.831109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767645.831111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767645.831113:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767645.831115:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3534 00000100:00080000:0.0:1775767645.831117:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767645.831119:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767645.831124:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767645.831124:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767645.831141:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742081792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767645.831166:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767645/real 1775767645] req@ffff90c203639500 x1862023742081408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767645.831187:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742081408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767645.831199:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767645.831204:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767645.831208:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742081408:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767645.831237:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081536:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767645.831255:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767645.831264:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767645/real 1775767645] req@ffff90c20363b100 x1862023742081536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767645.831272:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742081536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767645.831278:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767645.831281:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767645.831284:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767645.831316:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767645.831338:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767645.831351:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767645/real 1775767645] req@ffff90c20363a680 x1862023742081664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767645.831361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742081664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767645.831367:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767645.831369:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767645.831371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767645.831376:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081792:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767645.831390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767645.831397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767645/real 1775767645] req@ffff90c203638380 x1862023742081792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767645.831409:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742081792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767670 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767645.831420:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767645.831423:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767645.831427:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742081792:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767646.855035:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767647.878968:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767648.903016:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767649.927042:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767650.950971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767650.950979:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767650.950983:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767650.950991:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3539 00000100:00080000:1.0:1775767650.951000:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767650.951007:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767650.951043:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742081920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767650.951043:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767650.951068:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767650.951070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767650.951072:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767650.951075:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3539 00000100:00080000:1.0:1775767650.951077:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767650.951080:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767650.951087:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767650.951098:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742082048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767650.951107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767650.951107:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742081920:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767650.951114:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767650.951115:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767650.951118:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3539 00000100:00080000:1.0:1775767650.951120:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767650.951122:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767650.951126:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767650.951133:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742082176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767650.951139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767650.951141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767650.951142:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767650.951145:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3539 00000100:00080000:1.0:1775767650.951147:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767650.951149:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767650.951152:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767650.951158:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742082304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767650.951175:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767650.951214:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767650/real 1775767650] req@ffff90c203496d80 x1862023742081920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767650.951229:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742081920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767650.951239:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767650.951254:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767650.951258:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742081920:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767650.951281:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082048:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767650.951305:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767650.951317:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767650/real 1775767650] req@ffff90c203497b80 x1862023742082048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767650.951326:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742082048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767650.951332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767650.951334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767650.951336:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767650.951341:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082176:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767650.951354:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767650.951360:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767650/real 1775767650] req@ffff90c203496300 x1862023742082176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767650.951371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742082176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767650.951379:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767650.951382:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767650.951386:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767650.951393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082304:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767650.951414:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767650.951425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767650/real 1775767650] req@ffff90c203496a00 x1862023742082304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767650.951438:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742082304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767675 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767650.951448:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767650.951450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767650.951453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082304:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767651.975068:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767652.999060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767654.023076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767655.047040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767656.070972:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767656.070980:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767656.070984:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767656.070993:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3544 00000100:00080000:1.0:1775767656.071002:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767656.071008:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767656.071036:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767656.071047:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742082432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767656.071074:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767656.071078:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767656.071080:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767656.071085:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3544 00000100:00080000:1.0:1775767656.071088:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767656.071093:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767656.071104:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767656.071107:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742082432:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767656.071128:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742082560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767656.071139:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767656.071143:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767656.071145:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767656.071149:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3544 00000100:00080000:1.0:1775767656.071153:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767656.071157:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767656.071163:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775767656.071165:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767656.071178:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742082688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767656.071189:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767656.071207:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767656/real 1775767656] req@ffff90c203496680 x1862023742082432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767656.071209:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767656.071214:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767656.071224:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3544 00000100:00080000:1.0:1775767656.071230:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767656.071233:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742082432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767656.071237:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767656.071247:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775767656.071248:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767656.071255:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767656.071258:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742082432:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767656.071264:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742082816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767656.071279:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082560:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767656.071297:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767656.071305:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767656/real 1775767656] req@ffff90c203494a80 x1862023742082560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767656.071314:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742082560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767656.071320:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767656.071322:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767656.071324:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767656.071328:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767656.071337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767656.071342:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767656/real 1775767656] req@ffff90c203494700 x1862023742082688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767656.071349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742082688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767656.071354:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767656.071356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767656.071357:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767656.071361:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082816:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767656.071370:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767656.071375:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767656/real 1775767656] req@ffff90c203495500 x1862023742082816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767656.071381:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742082816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767656.071392:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767656.071394:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767656.071396:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742082816:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767657.095019:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767658.119053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767659.143008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767660.167096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767661.190962:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767661.190971:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767661.190976:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767661.190983:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3549 00000100:00080000:1.0:1775767661.190994:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767661.191001:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767661.191028:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767661.191040:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742082944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767661.191075:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767661.191078:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767661.191080:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767661.191083:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3549 00000100:00080000:1.0:1775767661.191087:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767661.191090:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767661.191099:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767661.191116:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742083072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767661.191120:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742082944:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767661.191136:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767661.191140:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767661.191166:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767661.191172:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3549 00000100:00080000:1.0:1775767661.191176:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767661.191180:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767661.191184:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767661.191194:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742083200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767661.191201:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767661.191204:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767661.191207:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767661.191209:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767661.191212:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3549 00000100:00080000:1.0:1775767661.191216:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767661.191218:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767661.191222:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767661.191229:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742083328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767661.191256:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767661/real 1775767661] req@ffff90c203495180 x1862023742082944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767661.191278:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742082944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767661.191289:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767661.191293:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767661.191298:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742082944:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767661.191334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767661.191357:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767661.191371:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767661/real 1775767661] req@ffff90c203495500 x1862023742083072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767661.191386:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742083072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767661.191395:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767661.191399:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767661.191403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767661.191410:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767661.191430:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767661.191442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767661/real 1775767661] req@ffff90c203494700 x1862023742083200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767661.191455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742083200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767661.191464:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767661.191467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767661.191470:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767661.191476:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083328:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767661.191496:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767661.191507:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767661/real 1775767661] req@ffff90c203494a80 x1862023742083328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767661.191520:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742083328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767661.191530:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767661.191534:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767661.191537:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083328:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767662.215085:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767663.238996:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767664.263042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767665.287035:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767666.311020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767666.311029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767666.311034:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767666.311042:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3554 00000100:00080000:2.0:1775767666.311052:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767666.311058:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767666.311086:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1775767666.311124:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bed80 x1862023742083456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767666.311154:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767666.311157:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767666.311158:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767666.311161:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3554 00000100:00080000:2.0:1775767666.311165:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767666.311168:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767666.311176:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767666.311187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be680 x1862023742083584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767666.311197:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742083456:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775767666.311197:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767666.311205:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767666.311208:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767666.311212:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3554 00000100:00080000:2.0:1775767666.311216:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767666.311220:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767666.311225:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767666.311241:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023742083712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767666.311255:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775767666.311265:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767666.311269:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767666.311271:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767666.311275:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3554 00000100:00080000:2.0:1775767666.311278:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767666.311281:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767666.311286:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767666.311291:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767666/real 1775767666] req@ffff90c2034bed80 x1862023742083456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767666.311298:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023742083840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767666.311333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bed80 x1862023742083456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767666.311345:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767666.311350:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767666.311355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742083456:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767666.311383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083584:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767666.311401:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767666.311410:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767666/real 1775767666] req@ffff90c2034be680 x1862023742083584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767666.311419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be680 x1862023742083584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767666.311425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767666.311427:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767666.311429:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767666.311432:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083712:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767666.311443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767666.311448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767666/real 1775767666] req@ffff90c2034bc700 x1862023742083712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767666.311455:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023742083712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767666.311464:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767666.311466:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767666.311468:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767666.311472:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083840:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767666.311481:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767666.311486:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767666/real 1775767666] req@ffff90c2034bf100 x1862023742083840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767666.311492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023742083840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767666.311498:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767666.311500:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767666.311502:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742083840:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767667.335112:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767668.359062:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767669.383050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767670.407008:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767671.431076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767671.431091:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767671.431098:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767671.431114:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3559 00000100:00080000:2.0:1775767671.431132:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767671.431142:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767671.431200:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023742083968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767671.431239:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767671.431243:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767671.431246:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767671.431251:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3559 00000100:00080000:2.0:1775767671.431255:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767671.431260:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767671.431275:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767671.431288:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742083968:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767671.431294:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023742084096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767671.431311:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767671.431315:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767671.431317:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767671.431322:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3559 00000100:00080000:2.0:1775767671.431327:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767671.431331:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767671.431337:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767671.431350:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023742084224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767671.431361:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775767671.431366:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775767671.431369:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767671.431372:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767671.431379:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3559 00000100:00080000:2.0:1775767671.431385:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767671.431389:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767671.431413:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767671.431428:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767671/real 1775767671] req@ffff90c2034bdf80 x1862023742083968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767671.431433:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023742084352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767671.431459:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023742083968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767671.431487:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767671.431493:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767671.431497:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742083968:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767671.431527:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767671.431560:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767671.431573:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767671/real 1775767671] req@ffff90c2034bd180 x1862023742084096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767671.431588:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023742084096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767671.431597:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767671.431601:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00010000:00010000:3.0:1775767671.431601:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767671.431608:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767671.431615:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767671.431631:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767671.431639:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767671/real 1775767671] req@ffff90c2034bc380 x1862023742084224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767671.431648:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023742084224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767671.431656:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767671.431660:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767671.431665:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767671.431673:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084352:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767671.431701:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767671.431716:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767671/real 1775767671] req@ffff90c2034bce00 x1862023742084352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767671.431734:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023742084352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767671.431748:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767671.431753:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767671.431757:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084352:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767672.455087:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767673.479132:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767674.503150:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767675.527176:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767676.550974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767676.550982:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767676.550987:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767676.551015:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3565 00000100:00080000:2.0:1775767676.551025:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767676.551031:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767676.551065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023742084480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767676.551089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767676.551092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767676.551093:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767676.551096:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3565 00000100:00080000:2.0:1775767676.551098:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767676.551100:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767676.551108:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767676.551117:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023742084608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767676.551124:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767676.551127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767676.551128:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767676.551131:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3565 00000100:00080000:2.0:1775767676.551133:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767676.551134:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767676.551138:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767676.551150:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742084736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767676.551151:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742084480:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775767676.551174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767676.551178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767676.551179:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767676.551184:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3565 00000100:00080000:2.0:1775767676.551188:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767676.551193:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767676.551198:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767676.551211:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742084864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767676.551217:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767676.551256:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767676/real 1775767676] req@ffff90c2034bf480 x1862023742084480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767676.551271:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023742084480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767676.551280:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767676.551283:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767676.551287:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742084480:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767676.551292:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767676.551308:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00010000:00010000:3.0:1775767676.551314:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767676.551319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767676/real 1775767676] req@ffff90c2034be300 x1862023742084608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767676.551332:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023742084608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767676.551347:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767676.551351:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767676.551355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767676.551379:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084736:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767676.551397:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767676.551405:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767676/real 1775767676] req@ffff90c2034bca80 x1862023742084736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767676.551412:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742084736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767676.551418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767676.551420:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767676.551422:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767676.551426:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084864:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767676.551436:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767676.551445:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767676/real 1775767676] req@ffff90c2034bea00 x1862023742084864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767676.551457:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742084864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767676.551466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767676.551469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767676.551474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742084864:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767677.575065:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767678.599073:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767679.623042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767680.647042:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767681.671048:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767681.671058:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767681.671063:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767681.671074:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3570 00000100:00080000:3.0:1775767681.671096:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767681.671104:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767681.671123:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767681.671141:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742084992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767681.671174:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767681.671179:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767681.671181:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767681.671185:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3570 00000100:00080000:3.0:1775767681.671189:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767681.671194:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767681.671205:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767681.671220:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742085120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767681.671221:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742084992:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767681.671238:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767681.671242:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767681.671244:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767681.671249:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3570 00000100:00080000:3.0:1775767681.671252:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767681.671256:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767681.671261:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767681.671272:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742085248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767681.671282:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767681.671285:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767681.671287:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767681.671290:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3570 00000100:00080000:3.0:1775767681.671293:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1775767681.671299:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767681.671301:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767681.671307:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767681.671317:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742085376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767681.671345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767681/real 1775767681] req@ffff90c218da2d80 x1862023742084992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767681.671360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742084992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767681.671368:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767681.671372:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767681.671375:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742084992:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767681.671392:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767681.671408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767681.671416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767681/real 1775767681] req@ffff90c218da3800 x1862023742085120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767681.671424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742085120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767681.671429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767681.671431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767681.671433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767681.671437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767681.671446:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767681.671462:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767681/real 1775767681] req@ffff90c218da1880 x1862023742085248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767681.671473:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742085248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767681.671478:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767681.671480:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767681.671482:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767681.671485:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085376:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767681.671496:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767681.671501:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767681/real 1775767681] req@ffff90c218da0e00 x1862023742085376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767706 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767681.671508:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742085376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767706 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767681.671514:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767681.671516:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767681.671517:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085376:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767682.695039:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767683.719073:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767684.743037:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767685.767098:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767686.791021:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767686.791031:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767686.791036:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767686.791044:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3575 00000100:00080000:1.0:1775767686.791055:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775767686.791060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767686.791061:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767686.791121:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742085504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767686.791138:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767686.791142:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767686.791143:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767686.791147:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3575 00000100:00080000:1.0:1775767686.791150:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767686.791153:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767686.791159:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742085504:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775767686.791163:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767686.791193:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742085632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767686.791203:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767686.791206:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767686.791208:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767686.791211:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3575 00000100:00080000:1.0:1775767686.791225:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767686.791229:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767686.791234:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775767686.791238:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767686.791250:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742085760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767686.791259:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767686.791262:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767686.791263:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767686.791267:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3575 00000100:00080000:1.0:1775767686.791270:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767686.791273:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767686.791277:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767686.791279:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767686/real 1775767686] req@ffff90c203496680 x1862023742085504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767686.791291:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742085888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767686.791301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742085504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767686.791312:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767686.791317:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767686.791322:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742085504:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767686.791352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767686.791371:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767686.791381:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767686/real 1775767686] req@ffff90c203496a00 x1862023742085632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767686.791391:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742085632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767686.791398:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767686.791400:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767686.791403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767686.791407:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767686.791419:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767686.791425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767686/real 1775767686] req@ffff90c203496300 x1862023742085760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767686.791434:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742085760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767686.791440:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767686.791443:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767686.791445:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767686.791450:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085888:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767686.791462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767686.791468:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767686/real 1775767686] req@ffff90c203497b80 x1862023742085888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767711 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767686.791477:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742085888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767711 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767686.791485:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767686.791487:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767686.791489:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742085888:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767687.815107:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767688.839090:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767689.863060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767690.887080:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767691.910982:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767691.910993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767691.910998:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767691.911008:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3580 00000100:00080000:1.0:1775767691.911020:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767691.911028:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767691.911052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767691.911068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742086016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767691.911104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767691.911109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767691.911111:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767691.911115:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3580 00000100:00080000:1.0:1775767691.911120:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767691.911124:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767691.911135:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767691.911147:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742086016:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767691.911153:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742086144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767691.911166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767691.911170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767691.911172:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767691.911176:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3580 00000100:00080000:1.0:1775767691.911181:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767691.911185:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767691.911191:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767691.911205:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742086272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767691.911215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767691.911219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767691.911221:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767691.911226:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3580 00000100:00080000:1.0:1775767691.911242:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767691.911247:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767691.911251:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086144:192.168.202.155@tcp:38:kworker.0 00000008:00100000:1.0:1775767691.911255:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767691.911272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742086400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767691.911281:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767691.911321:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086400:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767692.935064:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767693.959032:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767694.983130:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775767694.983154:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775767694.983193:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775767694.983212:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767694.983259:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767694.983268:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767694.983275:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767694.983343:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767691/real 1775767694] req@ffff90c203495c00 x1862023742086400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767694.983383:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742086400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767694.983401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767694.983409:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767694.983413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086400:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767694.983423:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767691/real 1775767694] req@ffff90c203495880 x1862023742086144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767694.983438:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742086144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767694.983446:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767694.983451:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767694.983455:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767694.983463:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767691/real 1775767694] req@ffff90c203497100 x1862023742086272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767694.983475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742086272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767694.983483:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767694.983487:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767694.983490:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767694.983497:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767691/real 1775767694] req@ffff90c203496d80 x1862023742086016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767694.983512:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742086016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767716 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767694.983520:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767694.983523:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767694.983526:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742086016:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767696.007079:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767697.031002:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767697.031013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767697.031018:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767697.031028:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3585 00000100:00080000:1.0:1775767697.031042:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767697.031050:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767697.031056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767697.031092:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742086528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767697.031116:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767697.031120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767697.031121:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767697.031124:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3585 00000100:00080000:1.0:1775767697.031126:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767697.031129:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767697.031138:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767697.031148:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742086656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767697.031157:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742086528:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767697.031159:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767697.031166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767697.031168:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767697.031171:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3585 00000100:00080000:1.0:1775767697.031173:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767697.031176:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767697.031179:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767697.031189:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742086784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767697.031196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767697.031198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767697.031199:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767697.031202:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3585 00000100:00080000:1.0:1775767697.031205:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767697.031208:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767697.031211:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767697.031218:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742086912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767697.031236:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767697.031273:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767697/real 1775767697] req@ffff90c203494000 x1862023742086528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767697.031290:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742086528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767697.031300:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767697.031303:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767697.031307:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742086528:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767697.031329:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767697.031344:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767697.031351:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767697/real 1775767697] req@ffff90c203494e00 x1862023742086656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767697.031360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742086656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767697.031369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767697.031372:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767697.031375:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767697.031382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767697.031403:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767697.031415:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767697/real 1775767697] req@ffff90c203495f80 x1862023742086784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767697.031425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742086784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767697.031433:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767697.031436:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767697.031439:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767697.031446:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086912:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767697.031466:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767697.031478:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767697/real 1775767697] req@ffff90c203494380 x1862023742086912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767697.031489:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742086912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767697.031500:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767697.031503:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767697.031506:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742086912:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767698.055042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767699.079085:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767700.103015:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767701.126997:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767702.150971:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767702.151020:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767702.151031:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767702.151037:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767702.151048:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3590 00000100:00080000:3.0:1775767702.151060:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767702.151068:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767702.151103:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742087040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767702.151124:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767702.151127:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767702.151128:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767702.151132:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3590 00000100:00080000:3.0:1775767702.151136:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767702.151140:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767702.151150:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767702.151152:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742087040:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767702.151162:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742087168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767702.151171:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767702.151173:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767702.151175:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767702.151177:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3590 00000100:00080000:3.0:1775767702.151179:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767702.151182:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767702.151185:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767702.151199:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742087296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767702.151206:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767702.151209:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767702.151213:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767702.151215:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767702.151220:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3590 00000100:00080000:3.0:1775767702.151224:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767702.151227:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767702.151231:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767702.151241:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742087424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767702.151247:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767702/real 1775767702] req@ffff90c218da0a80 x1862023742087040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767702.151265:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742087040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767702.151273:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767702.151276:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767702.151279:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742087040:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767702.151297:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767702.151324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767702.151337:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767702/real 1775767702] req@ffff90c218da0700 x1862023742087168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767702.151351:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742087168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767702.151361:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767702.151364:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767702.151368:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767702.151381:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767702.151398:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767702.151406:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767702/real 1775767702] req@ffff90c218da0380 x1862023742087296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767702.151414:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742087296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767702.151419:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767702.151422:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767702.151423:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767702.151427:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087424:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767702.151437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767702.151442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767702/real 1775767702] req@ffff90c218da3480 x1862023742087424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767702.151448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742087424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767702.151466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767702.151468:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767702.151471:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087424:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767703.174975:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767704.198990:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767705.223069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767706.247018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767707.270977:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767707.270988:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767707.270994:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767707.271005:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3595 00000100:00080000:3.0:1775767707.271017:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767707.271026:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767707.271068:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742087552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767707.271094:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767707.271097:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767707.271099:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767707.271114:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3595 00000100:00080000:3.0:1775767707.271120:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775767707.271121:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767707.271127:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767707.271136:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767707.271143:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742087552:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767707.271158:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742087680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767707.271167:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767707.271170:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767707.271171:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767707.271174:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3595 00000100:00080000:3.0:1775767707.271193:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767707.271197:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767707.271201:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767707.271203:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767707.271217:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742087808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767707.271228:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767707.271232:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767707.271234:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767707.271238:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767707/real 1775767707] req@ffff90c218da1f80 x1862023742087552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767707.271241:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3595 00000100:00080000:3.0:1775767707.271250:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767707.271256:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767707.271260:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742087552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1775767707.271264:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775767707.271271:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767707.271276:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767707.271280:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742087552:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767707.271284:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742087936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767707.271321:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767707.271348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767707.271369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767707/real 1775767707] req@ffff90c218da0000 x1862023742087680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767707.271382:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742087680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767707.271391:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767707.271394:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767707.271398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767707.271405:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087808:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767707.271425:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767707.271435:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767707/real 1775767707] req@ffff90c218da3100 x1862023742087808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767707.271448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742087808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767707.271456:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767707.271460:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767707.271463:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767707.271468:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087936:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767707.271485:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767707.271492:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767707/real 1775767707] req@ffff90c218da3480 x1862023742087936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767707.271499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742087936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767707.271506:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767707.271508:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767707.271510:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742087936:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767708.295062:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767709.319055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767710.343051:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767711.367058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767712.390969:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767712.391009:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767712.391015:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767712.391020:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767712.391027:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3600 00000100:00080000:0.0:1775767712.391037:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767712.391042:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767712.391075:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742088064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767712.391099:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767712.391103:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767712.391105:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767712.391109:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3600 00000100:00080000:0.0:1775767712.391113:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767712.391117:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767712.391127:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767712.391135:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742088064:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767712.391145:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742088192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767712.391158:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767712.391162:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767712.391164:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767712.391168:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3600 00000100:00080000:0.0:1775767712.391172:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767712.391175:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767712.391180:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767712.391186:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767712.391191:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742088320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767712.391203:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767712.391205:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767712.391206:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767712.391210:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3600 00000100:00080000:0.0:1775767712.391213:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767712.391215:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767712.391219:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767712.391224:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767712/real 1775767712] req@ffff90c203638e00 x1862023742088064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767712.391229:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742088448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767712.391262:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742088064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767712.391273:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767712.391276:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767712.391279:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742088064:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767712.391303:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767712.391327:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767712.391339:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767712/real 1775767712] req@ffff90c203638000 x1862023742088192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767712.391351:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742088192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767712.391357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767712.391359:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767712.391362:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767712.391366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767712.391379:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767712.391386:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767712/real 1775767712] req@ffff90c203639180 x1862023742088320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767712.391394:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742088320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767712.391404:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767712.391425:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767712.391429:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767712.391436:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088448:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767712.391459:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767712.391471:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767712/real 1775767712] req@ffff90c20363aa00 x1862023742088448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767712.391485:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742088448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767712.391494:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767712.391496:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767712.391499:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088448:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767713.415014:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767714.439056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767715.463030:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767716.487060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767717.510987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767717.510995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767717.510999:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767717.511008:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3605 00000100:00080000:1.0:1775767717.511017:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767717.511023:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767717.511040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767717.511060:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742088576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767717.511089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767717.511093:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767717.511095:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767717.511099:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3605 00000100:00080000:1.0:1775767717.511102:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767717.511105:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767717.511113:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767717.511128:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742088704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767717.511128:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742088576:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767717.511143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767717.511145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767717.511147:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767717.511150:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3605 00000100:00080000:1.0:1775767717.511153:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767717.511155:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767717.511158:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767717.511166:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742088832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767717.511177:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767717.511180:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767717.511181:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767717.511184:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3605 00000100:00080000:1.0:1775767717.511186:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767717.511189:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767717.511194:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767717.511194:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767717.511204:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742088960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767717.511241:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767717/real 1775767717] req@ffff90c203497480 x1862023742088576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767717.511262:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742088576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767717.511273:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767717.511278:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767717.511283:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742088576:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767717.511309:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767717.511332:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767717.511342:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767717/real 1775767717] req@ffff90c203494380 x1862023742088704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767717.511350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742088704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767717.511364:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767717.511370:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767717.511373:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767717.511377:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088832:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767717.511389:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767717.511395:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767717/real 1775767717] req@ffff90c203495f80 x1862023742088832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767717.511402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742088832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767717.511407:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767717.511409:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767717.511411:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767717.511414:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088960:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767717.511424:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767717.511429:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767717/real 1775767717] req@ffff90c203494e00 x1862023742088960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767717.511437:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742088960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767717.511443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767717.511445:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767717.511447:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742088960:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767718.535077:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767719.559006:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767720.583090:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767721.607002:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767722.630986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767722.630994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767722.630999:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767722.631008:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3611 00000100:00080000:1.0:1775767722.631018:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767722.631025:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767722.631061:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742089088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767722.631083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767722.631087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767722.631089:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767722.631093:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3611 00000100:00080000:1.0:1775767722.631096:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767722.631101:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767722.631116:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767722.631129:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742089088:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767722.631133:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742089216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767722.631133:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767722.631152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767722.631156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767722.631158:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767722.631165:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3611 00000100:00080000:1.0:1775767722.631170:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767722.631175:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767722.631181:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767722.631195:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742089344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767722.631199:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767722.631212:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767722.631216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767722.631218:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767722.631224:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3611 00000100:00080000:1.0:1775767722.631229:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767722.631234:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767722.631240:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767722.631248:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767722/real 1775767722] req@ffff90c203494000 x1862023742089088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767722.631255:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742089472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767722.631284:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742089088/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767722.631297:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767722.631302:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767722.631307:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742089088:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767722.631327:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089216:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767722.631345:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767722.631354:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767722/real 1775767722] req@ffff90c203496d80 x1862023742089216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767722.631362:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742089216/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767722.631373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767722.631376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767722.631378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767722.631381:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767722.631392:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767722.631397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767722/real 1775767722] req@ffff90c203497100 x1862023742089344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767722.631404:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742089344/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767722.631410:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767722.631412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767722.631413:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767722.631417:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089472:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767722.631427:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767722.631432:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767722/real 1775767722] req@ffff90c203495880 x1862023742089472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767747 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767722.631439:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742089472/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767747 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767722.631446:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767722.631448:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767722.631450:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089472:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767723.655023:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767724.679100:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767725.703117:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767726.727114:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767727.751000:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767727.751008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767727.751013:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767727.751022:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3616 00000100:00080000:1.0:1775767727.751033:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767727.751041:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767727.751079:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742089600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767727.751091:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767727.751108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767727.751119:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767727.751121:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767727.751127:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3616 00000100:00080000:1.0:1775767727.751133:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767727.751137:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767727.751145:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767727.751158:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742089728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767727.751168:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742089600:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767727.751185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767727.751190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767727.751192:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767727.751197:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3616 00000100:00080000:1.0:1775767727.751209:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767727.751222:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767727.751229:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767727.751245:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742089856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767727.751256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775767727.751261:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767727.751266:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767727.751269:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767727.751276:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3616 00000100:00080000:1.0:1775767727.751282:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767727.751291:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767727.751297:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767727.751322:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767727/real 1775767727] req@ffff90c203495c00 x1862023742089600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767727.751323:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742089984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767727.751369:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742089600/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767727.751385:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767727.751390:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767727.751396:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742089600:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767727.751431:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767727.751459:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767727.751470:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767727/real 1775767727] req@ffff90c203497b80 x1862023742089728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767727.751480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742089728/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767727.751488:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767727.751492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767727.751496:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767727.751515:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767727.751539:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767727.751552:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767727/real 1775767727] req@ffff90c203496300 x1862023742089856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767727.751566:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742089856/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767727.751572:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767727.751574:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767727.751576:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767727.751581:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089984:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767727.751598:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767727.751605:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767727/real 1775767727] req@ffff90c203496a00 x1862023742089984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767752 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767727.751612:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742089984/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767752 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767727.751620:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767727.751621:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767727.751623:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742089984:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767728.775074:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767729.799105:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767730.823035:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767731.847057:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767732.870981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767732.870989:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767732.870994:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767732.871002:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3621 00000100:00080000:1.0:1775767732.871017:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767732.871024:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767732.871028:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767732.871059:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742090112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767732.871078:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767732.871081:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767732.871082:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767732.871086:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3621 00000100:00080000:1.0:1775767732.871088:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767732.871090:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767732.871098:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767732.871107:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742090240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767732.871115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767732.871117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767732.871118:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767732.871121:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3621 00000100:00080000:1.0:1775767732.871123:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767732.871125:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767732.871132:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742090112:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775767732.871134:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767732.871145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742090368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767732.871151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767732.871154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767732.871155:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767732.871158:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3621 00000100:00080000:1.0:1775767732.871160:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767732.871164:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767732.871170:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767732.871180:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742090496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767732.871192:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767732.871224:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767732/real 1775767732] req@ffff90c203496680 x1862023742090112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767732.871237:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742090112/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767732.871248:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767732.871253:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767732.871257:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742090112:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767732.871264:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090240:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767732.871287:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767732.871299:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767732/real 1775767732] req@ffff90c203494a80 x1862023742090240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767732.871312:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742090240/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767732.871317:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767732.871319:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767732.871321:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767732.871340:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090368:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767732.871358:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767732.871368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767732/real 1775767732] req@ffff90c203494700 x1862023742090368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767732.871380:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742090368/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767732.871388:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767732.871392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767732.871395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767732.871416:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090496:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767732.871432:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767732.871440:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767732/real 1775767732] req@ffff90c203495500 x1862023742090496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767732.871471:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742090496/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767757 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767732.871479:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767732.871482:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767732.871484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090496:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767733.895097:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767734.919007:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767735.943047:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767736.967036:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767737.991002:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767737.991012:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767737.991018:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767737.991028:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3626 00000100:00080000:0.0:1775767737.991039:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767737.991047:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767737.991057:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767737.991085:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742090624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767737.991114:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767737.991118:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767737.991120:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767737.991124:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3626 00000100:00080000:0.0:1775767737.991128:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767737.991132:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767737.991143:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767737.991158:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742090752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767737.991166:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742090624:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767737.991175:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767737.991179:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767737.991182:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767737.991187:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3626 00000100:00080000:0.0:1775767737.991191:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767737.991195:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767737.991201:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767737.991212:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742090880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767737.991223:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767737.991227:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767737.991230:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767737.991235:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767737.991239:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3626 00000100:00080000:0.0:1775767737.991243:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767737.991248:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767737.991255:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767737.991268:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742091008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767737.991280:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767737/real 1775767737] req@ffff90c203638700 x1862023742090624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767737.991315:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742090624/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767737.991327:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767737.991332:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767737.991337:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742090624:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767737.991360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767737.991384:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767737.991396:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767737/real 1775767737] req@ffff90c20363aa00 x1862023742090752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767737.991428:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742090752/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767737.991438:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767737.991442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767737.991445:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767737.991452:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767737.991472:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767737.991483:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767737/real 1775767737] req@ffff90c203639180 x1862023742090880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767737.991495:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742090880/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767737.991503:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767737.991507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767737.991510:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742090880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767737.991527:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091008:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767737.991547:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767737.991557:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767737/real 1775767737] req@ffff90c203638000 x1862023742091008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767737.991569:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742091008/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767762 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767737.991586:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767737.991590:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767737.991594:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091008:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767739.015050:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767740.039050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767741.063005:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767742.087072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767743.110936:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767743.110943:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767743.110948:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767743.110956:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3631 00000100:00080000:0.0:1775767743.110965:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767743.110971:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767743.111008:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742091136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775767743.111010:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767743.111044:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767743.111047:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767743.111048:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767743.111051:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3631 00000100:00080000:0.0:1775767743.111054:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767743.111057:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767743.111065:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767743.111077:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742091264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767743.111086:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767743.111087:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742091136:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767743.111090:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767743.111093:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767743.111099:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3631 00000100:00080000:0.0:1775767743.111101:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767743.111103:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767743.111106:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767743.111114:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742091392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767743.111125:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767743.111133:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767743.111134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767743.111136:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3631 00000100:00080000:0.0:1775767743.111139:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767743.111141:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767743.111145:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767743.111152:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742091520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767743.111158:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767743.111201:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767743/real 1775767743] req@ffff90c203638e00 x1862023742091136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767743.111221:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742091136/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767743.111231:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767743.111236:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767743.111240:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742091136:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767743.111264:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767743.111288:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767743.111300:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767743/real 1775767743] req@ffff90c203638380 x1862023742091264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767743.111314:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742091264/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767743.111323:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767743.111327:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767743.111330:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767743.111338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091392:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767743.111390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767743.111402:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767743/real 1775767743] req@ffff90c20363a680 x1862023742091392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767743.111415:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742091392/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767743.111424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767743.111428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767743.111431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767743.111438:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091520:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767743.111457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767743.111467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767743/real 1775767743] req@ffff90c20363b100 x1862023742091520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767743.111480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742091520/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767743.111491:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767743.111495:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767743.111498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091520:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767744.135052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767745.159068:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767746.183054:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767747.207036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767748.230997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767748.231007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767748.231012:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767748.231030:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3636 00000100:00080000:3.0:1775767748.231043:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767748.231050:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767748.231051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767748.231088:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742091648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767748.231109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767748.231113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767748.231116:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767748.231121:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3636 00000100:00080000:3.0:1775767748.231125:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767748.231129:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767748.231140:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767748.231146:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742091648:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767748.231168:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742091776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767748.231182:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767748.231186:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767748.231188:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767748.231192:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3636 00000100:00080000:3.0:1775767748.231195:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767748.231198:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767748.231203:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775767748.231203:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767748.231214:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742091904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767748.231232:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767748.231235:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767748.231237:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767748.231239:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767748/real 1775767748] req@ffff90c218da0380 x1862023742091648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767748.231241:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3636 00000100:00080000:3.0:1775767748.231245:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767748.231249:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767748.231253:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767748.231258:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742091648/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767748.231265:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742092032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767748.231268:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767748.231273:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767748.231277:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742091648:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767748.231329:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091776:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767748.231353:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767748.231364:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767748/real 1775767748] req@ffff90c218da0700 x1862023742091776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767748.231373:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742091776/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767748.231379:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767748.231381:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767748.231384:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767748.231387:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091904:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767748.231398:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767748.231404:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767748/real 1775767748] req@ffff90c218da0a80 x1862023742091904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767748.231411:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742091904/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767748.231424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767748.231428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767748.231431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742091904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767748.231437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092032:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767748.231456:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767748.231467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767748/real 1775767748] req@ffff90c218da0e00 x1862023742092032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767748.231480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742092032/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767748.231501:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767748.231503:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767748.231506:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092032:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767749.255047:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767750.279037:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767751.303055:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767752.327043:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767753.350991:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767753.350999:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767753.351003:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767753.351011:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3641 00000100:00080000:0.0:1775767753.351021:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767753.351028:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767753.351031:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767753.351063:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742092160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767753.351084:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767753.351087:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767753.351088:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767753.351091:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3641 00000100:00080000:0.0:1775767753.351093:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767753.351096:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767753.351105:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767753.351115:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742092288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767753.351119:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742092160:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767753.351131:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767753.351133:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767753.351134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767753.351137:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3641 00000100:00080000:0.0:1775767753.351139:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767753.351142:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767753.351153:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767753.351161:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742092416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767753.351168:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767753.351170:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767753.351171:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767753.351175:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3641 00000800:00000100:2.0:1775767753.351179:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767753.351181:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767753.351185:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767753.351190:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767753.351198:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742092544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767753.351226:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767753/real 1775767753] req@ffff90c203639500 x1862023742092160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767753.351239:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742092160/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767753.351247:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767753.351272:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767753.351276:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742092160:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767753.351297:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767753.351315:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767753.351323:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767753/real 1775767753] req@ffff90c20363b480 x1862023742092288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767753.351332:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742092288/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767753.351340:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767753.351343:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767753.351347:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767753.351353:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767753.351372:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767753.351383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767753/real 1775767753] req@ffff90c203639f80 x1862023742092416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767753.351395:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742092416/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767753.351403:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767753.351407:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767753.351411:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092416:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767753.351417:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092544:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767753.351437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767753.351445:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767753/real 1775767753] req@ffff90c20363bb80 x1862023742092544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767753.351453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742092544/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767753.351460:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767753.351462:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767753.351464:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092544:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767754.375008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767755.398993:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767756.423047:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767757.447052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767758.470996:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767758.471007:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767758.471012:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767758.471023:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3646 00000100:00080000:3.0:1775767758.471036:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767758.471044:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767758.471055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767758.471090:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742092672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767758.471116:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767758.471119:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767758.471120:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767758.471123:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3646 00000100:00080000:3.0:1775767758.471125:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767758.471128:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767758.471138:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767758.471148:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742092800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767758.471158:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742092672:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767758.471174:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767758.471178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767758.471179:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767758.471182:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3646 00000100:00080000:3.0:1775767758.471184:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767758.471186:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767758.471190:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767758.471217:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742092928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767758.471236:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767758.471238:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767758.471239:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767758.471243:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3646 00000100:00080000:3.0:1775767758.471249:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767758.471252:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767758.471258:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767758.471270:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742093056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767758.471308:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767758.471334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767758.471348:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093056:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767759.495075:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767760.518993:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767761.543090:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775767761.543207:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775767761.543252:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767761.543276:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767761.543324:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767761.543334:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767761.543342:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767761.543403:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767758/real 1775767761] req@ffff90c218da3800 x1862023742092800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767761.543436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742092800/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767761.543448:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767761.543453:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767761.543457:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767761.543464:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767758/real 1775767761] req@ffff90c218da2d80 x1862023742092928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767761.543477:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742092928/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767761.543486:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767761.543489:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767761.543493:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742092928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767761.543500:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767758/real 1775767761] req@ffff90c218da0e00 x1862023742093056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767761.543513:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742093056/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767761.543523:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767761.543527:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767761.543530:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093056:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767761.543538:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767758/real 1775767761] req@ffff90c218da1880 x1862023742092672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767761.543551:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742092672/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767761.543559:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767761.543562:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767761.543566:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742092672:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767762.567003:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767763.590989:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767763.590999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767763.591004:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767763.591015:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3652 00000100:00080000:3.0:1775767763.591028:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775767763.591031:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767763.591036:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767763.591079:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742093184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767763.591106:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767763.591109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767763.591110:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767763.591113:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3652 00000100:00080000:3.0:1775767763.591115:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767763.591117:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767763.591126:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767763.591135:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742093312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767763.591142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767763.591144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767763.591145:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767763.591164:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3652 00000100:00080000:3.0:1775767763.591168:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767763.591173:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767763.591178:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767763.591191:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742093440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767763.591204:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767763.591205:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742093184:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767763.591208:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767763.591211:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767763.591220:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3652 00000100:00080000:3.0:1775767763.591224:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767763.591228:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767763.591234:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767763.591254:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742093568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767763.591312:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767763.591361:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767763/real 1775767763] req@ffff90c218da0a80 x1862023742093184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767763.591385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742093184/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767763.591396:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767763.591401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767763.591406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742093184:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767763.591414:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767763.591439:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767763.591453:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767763/real 1775767763] req@ffff90c218da0700 x1862023742093312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767763.591466:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742093312/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767763.591476:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767763.591480:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767763.591483:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767763.591521:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767763.591544:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767763.591556:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767763/real 1775767763] req@ffff90c218da0380 x1862023742093440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767763.591568:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742093440/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767763.591582:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767763.591586:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767763.591588:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767763.591594:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093568:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767763.591615:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767763.591624:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767763/real 1775767763] req@ffff90c218da3480 x1862023742093568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767763.591634:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742093568/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767763.591645:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767763.591647:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767763.591650:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093568:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767764.615119:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767765.639057:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767766.663053:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767767.687016:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767768.710948:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767768.710956:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767768.710960:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767768.710968:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3657 00010000:00010000:3.0:1775767768.710974:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767768.710980:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767768.710988:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767768.711025:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742093696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767768.711053:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767768.711056:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767768.711058:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767768.711063:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3657 00000100:00080000:1.0:1775767768.711068:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767768.711072:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767768.711083:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767768.711088:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742093696:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767768.711115:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742093824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767768.711128:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767768.711132:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767768.711134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767768.711138:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3657 00000100:00080000:1.0:1775767768.711141:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767768.711144:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767768.711149:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767768.711160:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742093952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767768.711163:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767768.711171:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767768.711173:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767768.711174:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767768.711178:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3657 00000100:00080000:1.0:1775767768.711180:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767768.711183:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767768.711186:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767768.711194:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742094080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767768.711201:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767768/real 1775767768] req@ffff90c203495180 x1862023742093696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767768.711217:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742093696/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767768.711227:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767768.711232:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767768.711236:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742093696:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767768.711259:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767768.711284:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767768.711294:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767768/real 1775767768] req@ffff90c203495500 x1862023742093824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767768.711303:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742093824/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767768.711309:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767768.711312:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767768.711314:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767768.711318:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767768.711330:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767768.711336:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767768/real 1775767768] req@ffff90c203494700 x1862023742093952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767768.711343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742093952/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767768.711348:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767768.711350:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767768.711352:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742093952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767768.711355:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094080:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767768.711364:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767768.711369:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767768/real 1775767768] req@ffff90c203494a80 x1862023742094080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767793 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767768.711376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742094080/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767793 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767768.711383:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767768.711384:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767768.711386:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094080:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767769.735041:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767770.759050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767771.782999:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767772.807032:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767773.831006:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767773.831018:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767773.831023:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767773.831034:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3662 00010000:00010000:1.0:1775767773.831035:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767773.831083:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767773.831090:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767773.831125:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742094208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767773.831147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767773.831150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767773.831151:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767773.831154:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3662 00000100:00080000:3.0:1775767773.831157:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767773.831159:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767773.831168:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767773.831177:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742094336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767773.831185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767773.831189:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742094208:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767773.831189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767773.831190:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767773.831194:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3662 00000100:00080000:3.0:1775767773.831199:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767773.831201:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767773.831205:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767773.831214:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742094464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767773.831220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767773.831222:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767773.831224:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767773.831226:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3662 00000100:00080000:3.0:1775767773.831229:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767773.831231:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767773.831234:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767773.831242:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742094592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767773.831270:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767773.831300:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767773/real 1775767773] req@ffff90c218da3100 x1862023742094208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767773.831314:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742094208/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767773.831321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767773.831324:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767773.831327:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742094208:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767773.831344:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094336:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767773.831357:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767773.831364:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767773/real 1775767773] req@ffff90c218da0000 x1862023742094336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767773.831371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742094336/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767773.831382:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767773.831384:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767773.831386:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767773.831389:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094464:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767773.831399:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767773.831404:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767773/real 1775767773] req@ffff90c218da1f80 x1862023742094464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767773.831411:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742094464/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767773.831416:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767773.831418:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767773.831420:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767773.831423:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094592:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767773.831432:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767773.831437:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767773/real 1775767773] req@ffff90c218da3480 x1862023742094592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767773.831443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742094592/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767798 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767773.831449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767773.831451:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767773.831454:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094592:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767774.855068:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767775.879009:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767776.903070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767777.927032:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767778.950973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767778.950981:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767778.950986:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767778.950994:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3667 00000100:00080000:3.0:1775767778.951015:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767778.951022:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767778.951059:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742094720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775767778.951080:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767778.951080:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767778.951087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767778.951089:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767778.951092:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3667 00000100:00080000:3.0:1775767778.951096:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767778.951101:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767778.951109:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767778.951118:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742094848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767778.951130:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742094720:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767778.951134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767778.951140:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767778.951142:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767778.951146:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3667 00000100:00080000:3.0:1775767778.951148:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767778.951150:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767778.951153:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767778.951161:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742094976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767778.951168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767778.951170:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767778.951171:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767778.951174:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3667 00000100:00080000:3.0:1775767778.951177:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767778.951179:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767778.951182:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767778.951190:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742095104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767778.951212:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767778.951253:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767778/real 1775767778] req@ffff90c218da0380 x1862023742094720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767778.951275:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742094720/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767778.951287:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767778.951291:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767778.951296:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742094720:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767778.951321:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767778.951347:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767778.951361:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767778/real 1775767778] req@ffff90c218da0700 x1862023742094848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767778.951375:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742094848/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767778.951385:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767778.951389:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767778.951393:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767778.951400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767778.951433:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767778.951446:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767778/real 1775767778] req@ffff90c218da0a80 x1862023742094976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767778.951460:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742094976/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767778.951468:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767778.951471:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767778.951475:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742094976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767778.951481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095104:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767778.951502:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767778.951514:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767778/real 1775767778] req@ffff90c218da1880 x1862023742095104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767778.951526:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742095104/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767803 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767778.951537:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767778.951540:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767778.951544:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095104:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767779.975018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767780.999052:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767782.022994:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767783.047103:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767784.070989:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767784.070997:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767784.071002:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767784.071010:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3672 00000100:00080000:1.0:1775767784.071020:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775767784.071028:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767784.071030:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767784.071070:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742095232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767784.071091:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767784.071094:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767784.071095:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767784.071097:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3672 00000100:00080000:1.0:1775767784.071100:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767784.071103:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767784.071110:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767784.071118:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742095360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767784.071127:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767784.071128:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742095232:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767784.071132:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767784.071136:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767784.071139:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3672 00000100:00080000:1.0:1775767784.071141:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767784.071143:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767784.071147:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767784.071154:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742095488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767784.071160:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767784.071163:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767784.071164:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767784.071167:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3672 00000100:00080000:1.0:1775767784.071169:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767784.071171:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767784.071175:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767784.071185:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742095616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767784.071185:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767784.071226:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767784/real 1775767784] req@ffff90c203496680 x1862023742095232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767784.071247:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742095232/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767784.071258:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767784.071262:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767784.071267:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742095232:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767784.071301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767784.071325:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767784.071338:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767784/real 1775767784] req@ffff90c203496a00 x1862023742095360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767784.071351:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742095360/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767784.071361:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767784.071365:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767784.071368:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767784.071375:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095488:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767784.071394:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767784.071415:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767784/real 1775767784] req@ffff90c203496300 x1862023742095488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767784.071429:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742095488/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767784.071438:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767784.071442:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767784.071446:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767784.071453:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095616:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767784.071475:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767784.071487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767784/real 1775767784] req@ffff90c203497b80 x1862023742095616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767784.071501:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742095616/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767784.071512:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767784.071516:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767784.071520:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095616:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767785.095087:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767786.119044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767787.143107:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767788.167086:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767789.191004:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767789.191015:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767789.191022:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767789.191032:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3677 00010000:00010000:1.0:1775767789.191036:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767789.191044:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767789.191052:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767789.191089:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742095744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767789.191113:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767789.191117:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767789.191119:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767789.191124:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3677 00000100:00080000:3.0:1775767789.191128:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767789.191133:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767789.191144:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767789.191147:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742095744:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767789.191173:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742095872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767789.191186:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767789.191190:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767789.191192:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767789.191196:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3677 00000100:00080000:3.0:1775767789.191201:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767789.191205:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767789.191211:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767789.191212:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767789.191227:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742096000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767789.191245:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767789.191249:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767789.191251:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767789.191260:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767789/real 1775767789] req@ffff90c218da0e00 x1862023742095744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767789.191273:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3677 00000100:00080000:3.0:1775767789.191278:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767789.191279:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742095744/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767789.191287:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775767789.191294:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775767789.191297:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775767789.191299:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767789.191305:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742095744:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767789.191317:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742096128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767789.191326:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095872:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767789.191348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767789.191358:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767789/real 1775767789] req@ffff90c218da2d80 x1862023742095872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767789.191367:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742095872/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767789.191373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767789.191375:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767789.191377:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742095872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767789.191381:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767789.191390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767789.191396:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767789/real 1775767789] req@ffff90c218da3800 x1862023742096000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767789.191403:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742096000/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767789.191408:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767789.191410:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767789.191412:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767789.191421:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096128:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767789.191430:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767789.191435:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767789/real 1775767789] req@ffff90c218da1880 x1862023742096128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767789.191442:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742096128/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767789.191453:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767789.191456:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767789.191458:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096128:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767790.215076:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767791.239037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767792.263094:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767793.287006:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767794.311021:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767794.311031:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767794.311037:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767794.311048:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3682 00000100:00080000:3.0:1775767794.311062:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767794.311071:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767794.311105:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767794.311112:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742096256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767794.311137:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767794.311140:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767794.311142:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767794.311145:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3682 00000100:00080000:3.0:1775767794.311150:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767794.311154:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767794.311162:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767794.311172:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742096384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767794.311180:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742096256:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767794.311195:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767794.311198:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767794.311199:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767794.311203:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3682 00000100:00080000:3.0:1775767794.311205:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767794.311207:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767794.311211:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767794.311224:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742096512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767794.311231:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767794.311234:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767794.311236:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767794.311239:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3682 00000100:00080000:3.0:1775767794.311260:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767794.311263:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767794.311267:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767794.311278:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742096640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767794.311283:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767794.311337:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767794/real 1775767794] req@ffff90c218da0a80 x1862023742096256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767794.311354:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742096256/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767794.311362:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767794.311366:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767794.311369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742096256:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767794.311392:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767794.311408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767794.311416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767794/real 1775767794] req@ffff90c218da0700 x1862023742096384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767794.311424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742096384/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767794.311429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767794.311431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767794.311433:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767794.311438:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767794.311448:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767794.311454:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767794/real 1775767794] req@ffff90c218da0380 x1862023742096512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767794.311461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742096512/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767794.311465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767794.311467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767794.311476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767794.311481:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096640:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767794.311492:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767794.311498:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767794/real 1775767794] req@ffff90c218da3480 x1862023742096640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767794.311505:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742096640/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767794.311512:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767794.311514:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767794.311516:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096640:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767795.334985:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767796.359029:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767797.382986:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767798.407034:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767799.430950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767799.430959:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767799.430970:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767799.430981:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3687 00000100:00080000:3.0:1775767799.430992:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767799.430998:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767799.431001:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767799.431032:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742096768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767799.431053:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767799.431056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767799.431057:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767799.431060:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3687 00000100:00080000:3.0:1775767799.431062:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767799.431065:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767799.431072:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767799.431082:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742096896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767799.431090:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767799.431092:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742096768:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767799.431094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767799.431097:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767799.431102:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3687 00000100:00080000:3.0:1775767799.431104:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767799.431106:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767799.431110:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767799.431119:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742097024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767799.431126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767799.431129:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767799.431130:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767799.431133:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3687 00000100:00080000:3.0:1775767799.431142:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767799.431144:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767799.431148:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767799.431151:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767799.431161:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742097152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767799.431186:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767799/real 1775767799] req@ffff90c218da1f80 x1862023742096768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767799.431201:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742096768/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767799.431209:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767799.431212:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767799.431215:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742096768:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767799.431235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096896:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767799.431256:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767799.431268:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767799/real 1775767799] req@ffff90c218da0000 x1862023742096896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767799.431282:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742096896/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767799.431291:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767799.431295:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767799.431298:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742096896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767799.431302:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097024:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767799.431318:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767799.431325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767799/real 1775767799] req@ffff90c218da3100 x1862023742097024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767799.431333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742097024/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767799.431343:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767799.431345:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767799.431347:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767799.431351:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097152:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767799.431360:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767799.431366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767799/real 1775767799] req@ffff90c218da3480 x1862023742097152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767799.431377:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742097152/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767799.431387:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767799.431391:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767799.431394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097152:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767800.455033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767801.478980:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767802.503106:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767803.527058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767804.550986:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767804.550997:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767804.551002:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767804.551013:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3693 00000100:00080000:3.0:1775767804.551026:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767804.551033:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767804.551056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767804.551081:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742097280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767804.551106:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767804.551109:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767804.551111:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767804.551114:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3693 00000100:00080000:3.0:1775767804.551117:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767804.551120:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767804.551130:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767804.551140:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742097408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767804.551145:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742097280:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767804.551155:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767804.551157:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767804.551158:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767804.551162:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3693 00000100:00080000:3.0:1775767804.551163:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767804.551165:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767804.551169:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767804.551177:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742097536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767804.551184:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767804.551186:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767804.551187:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767804.551190:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3693 00000100:00080000:3.0:1775767804.551210:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767804.551213:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767804.551217:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767804.551223:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767804.551229:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742097664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767804.551268:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767804/real 1775767804] req@ffff90c218da0380 x1862023742097280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767804.551283:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742097280/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767804.551291:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767804.551295:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767804.551298:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742097280:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767804.551320:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767804.551335:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767804.551342:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767804/real 1775767804] req@ffff90c218da0700 x1862023742097408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767804.551350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742097408/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767804.551355:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767804.551357:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767804.551359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767804.551363:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097536:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767804.551372:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767804.551377:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767804/real 1775767804] req@ffff90c218da0a80 x1862023742097536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767804.551384:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742097536/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767804.551389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767804.551391:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767804.551392:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767804.551396:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097664:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767804.551405:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767804.551410:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767804/real 1775767804] req@ffff90c218da1880 x1862023742097664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767804.551417:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742097664/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767804.551424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767804.551426:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767804.551427:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097664:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767805.575129:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767806.599105:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767807.623041:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767808.647060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767809.671011:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767809.671021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767809.671028:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767809.671041:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3698 00000100:00080000:0.0:1775767809.671056:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767809.671065:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767809.671088:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767809.671117:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742097792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767809.671143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767809.671146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767809.671147:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767809.671150:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3698 00000100:00080000:0.0:1775767809.671162:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767809.671165:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767809.671175:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767809.671184:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742097792:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767809.671190:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742097920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767809.671199:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767809.671203:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767809.671204:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767809.671207:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3698 00000100:00080000:0.0:1775767809.671210:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767809.671212:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767809.671215:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767809.671226:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742098048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767809.671242:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767809.671246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767809.671248:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767809.671253:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3698 00000100:00080000:0.0:1775767809.671258:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775767809.671262:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775767809.671265:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767809.671272:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767809.671286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742098176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767809.671316:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767809/real 1775767809] req@ffff90c20363b800 x1862023742097792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767809.671339:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742097792/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767809.671348:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767809.671352:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767809.671355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742097792:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767809.671378:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767809.671401:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767809.671415:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767809/real 1775767809] req@ffff90c20363bb80 x1862023742097920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767809.671428:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742097920/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767809.671444:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767809.671448:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767809.671452:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742097920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767809.671456:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098048:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767809.671472:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767809.671480:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767809/real 1775767809] req@ffff90c203639f80 x1862023742098048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767809.671492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742098048/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767809.671500:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767809.671503:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767809.671506:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767809.671513:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098176:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767809.671533:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767809.671544:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767809/real 1775767809] req@ffff90c20363b480 x1862023742098176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767834 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767809.671553:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742098176/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767834 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767809.671560:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767809.671562:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767809.671565:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098176:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767810.695063:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767811.719053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767812.743071:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767813.767067:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767814.791014:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767814.791025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767814.791030:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767814.791042:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3703 00000100:00080000:0.0:1775767814.791054:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767814.791060:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767814.791089:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767814.791096:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742098304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767814.791131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767814.791135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767814.791137:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767814.791142:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3703 00000100:00080000:0.0:1775767814.791146:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767814.791150:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767814.791161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767814.791175:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742098432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767814.791185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767814.791189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767814.791193:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767814.791195:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742098304:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767814.791199:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3703 00000100:00080000:0.0:1775767814.791208:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767814.791213:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767814.791216:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767814.791225:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742098560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767814.791232:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767814.791235:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767814.791237:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767814.791241:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3703 00000100:00080000:0.0:1775767814.791243:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767814.791245:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767814.791248:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767814.791260:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742098688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767814.791302:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767814.791344:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767814/real 1775767814] req@ffff90c203639500 x1862023742098304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767814.791360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742098304/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767814.791369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767814.791372:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767814.791376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742098304:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767814.791399:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098432:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767814.791414:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767814.791422:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767814/real 1775767814] req@ffff90c20363b100 x1862023742098432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767814.791429:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742098432/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767814.791435:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767814.791437:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767814.791438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767814.791443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098560:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767814.791453:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767814.791459:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767814/real 1775767814] req@ffff90c20363a680 x1862023742098560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767814.791466:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742098560/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767814.791471:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767814.791472:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767814.791474:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767814.791478:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098688:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767814.791488:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767814.791493:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767814/real 1775767814] req@ffff90c203638380 x1862023742098688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767839 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767814.791500:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742098688/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767839 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767814.791507:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767814.791509:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767814.791510:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098688:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767815.815092:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767816.838999:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767817.863130:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767818.887051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767819.910994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767819.911003:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767819.911008:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767819.911024:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3708 00000100:00080000:1.0:1775767819.911036:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767819.911043:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767819.911052:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767819.911082:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742098816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767819.911107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767819.911113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767819.911116:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767819.911121:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3708 00000100:00080000:1.0:1775767819.911126:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767819.911131:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767819.911138:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742098816:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775767819.911142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767819.911159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742098944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767819.911170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767819.911172:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767819.911174:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767819.911176:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3708 00000100:00080000:1.0:1775767819.911179:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767819.911183:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767819.911187:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767819.911193:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767819.911197:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742099072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767819.911207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767819.911211:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767819.911213:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767819.911217:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3708 00000100:00080000:1.0:1775767819.911221:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767819.911226:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767819/real 1775767819] req@ffff90c203495c00 x1862023742098816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775767819.911227:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767819.911237:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767819.911245:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742098816/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775767819.911253:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742099200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767819.911256:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767819.911263:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767819.911270:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742098816:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767819.911292:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767819.911307:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767819.911315:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767819/real 1775767819] req@ffff90c203495880 x1862023742098944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767819.911323:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742098944/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767819.911329:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767819.911332:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767819.911334:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742098944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767819.911337:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767819.911347:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767819.911352:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767819/real 1775767819] req@ffff90c203497100 x1862023742099072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767819.911359:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742099072/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767819.911365:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767819.911367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767819.911368:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767819.911373:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099200:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767819.911383:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767819.911391:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767819/real 1775767819] req@ffff90c203496d80 x1862023742099200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767819.911403:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742099200/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767844 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767819.911413:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767819.911417:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767819.911421:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099200:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767820.935023:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767821.958997:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767822.983061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767824.007022:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767825.030966:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767825.030975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767825.030979:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767825.030987:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3713 00000100:00080000:1.0:1775767825.030997:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767825.031004:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767825.031033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767825.031045:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742099328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767825.031075:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767825.031078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767825.031080:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767825.031083:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3713 00000100:00080000:1.0:1775767825.031087:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767825.031091:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767825.031100:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767825.031108:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742099328:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767825.031114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742099456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767825.031122:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767825.031126:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767825.031127:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767825.031130:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3713 00000100:00080000:1.0:1775767825.031132:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767825.031134:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767825.031137:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767825.031145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742099584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767825.031151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767825.031154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767825.031156:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767825.031160:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3713 00000100:00080000:1.0:1775767825.031164:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767825.031168:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767825.031173:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767825.031185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742099712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767825.031207:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767825.031234:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767825.031249:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099712:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767826.055035:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767827.079093:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767828.103197:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775767828.103345:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775767828.103416:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775767828.103440:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767828.103511:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767828.103549:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775767828.103567:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767828.103594:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767825/real 1775767828] req@ffff90c203494e00 x1862023742099456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767828.103640:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742099456/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767828.103657:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767828.103665:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767828.103671:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767828.103691:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767825/real 1775767828] req@ffff90c203495f80 x1862023742099584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767828.103707:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742099584/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767828.103724:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767828.103729:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767828.103737:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767828.103746:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767825/real 1775767828] req@ffff90c203494380 x1862023742099712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767828.103760:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742099712/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767828.103772:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767828.103777:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767828.103781:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099712:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767828.103790:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767825/real 1775767828] req@ffff90c203494000 x1862023742099328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767828.103804:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742099328/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767828.103812:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767828.103817:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767828.103820:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742099328:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767829.127051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767830.151007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767830.151018:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767830.151024:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767830.151035:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3718 00000100:00080000:0.0:1775767830.151049:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767830.151056:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767830.151068:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767830.151098:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742099840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767830.151134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767830.151137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767830.151139:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767830.151142:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3718 00000100:00080000:0.0:1775767830.151144:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767830.151147:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767830.151158:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767830.151168:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742099968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767830.151178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767830.151178:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742099840:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767830.151184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767830.151188:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767830.151192:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3718 00000100:00080000:0.0:1775767830.151194:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767830.151197:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767830.151200:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767830.151207:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742100096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767830.151220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767830.151224:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767830.151226:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767830.151231:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3718 00000100:00080000:0.0:1775767830.151236:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767830.151241:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767830.151247:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767830.151261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742100224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767830.151268:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767830.151316:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767830/real 1775767830] req@ffff90c203638e00 x1862023742099840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767830.151338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742099840/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767830.151351:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767830.151356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767830.151360:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742099840:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767830.151394:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767830.151418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767830.151432:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767830/real 1775767830] req@ffff90c203638000 x1862023742099968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767830.151442:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742099968/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767830.151449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767830.151453:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767830.151457:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742099968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767830.151463:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767830.151505:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767830.151546:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767830/real 1775767830] req@ffff90c203639180 x1862023742100096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767830.151557:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742100096/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767830.151566:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767830.151569:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767830.151572:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767830.151578:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100224:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767830.151602:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767830.151613:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767830/real 1775767830] req@ffff90c20363aa00 x1862023742100224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767830.151626:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742100224/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767830.151635:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767830.151638:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767830.151642:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100224:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767831.175078:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767832.199051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767833.223071:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767834.247010:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767835.270987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767835.271002:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767835.271008:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767835.271028:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3723 00000100:00080000:0.0:1775767835.271046:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767835.271056:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767835.271066:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767835.271110:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742100352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767835.271143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767835.271147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767835.271149:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767835.271154:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3723 00000100:00080000:0.0:1775767835.271159:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767835.271163:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767835.271176:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767835.271179:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742100352:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767835.271195:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742100480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767835.271215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767835.271219:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767835.271221:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767835.271226:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3723 00000100:00080000:0.0:1775767835.271231:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767835.271235:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767835.271242:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767835.271251:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767835.271258:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742100608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767835.271270:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767835.271274:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767835.271275:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767835.271279:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3723 00000100:00080000:0.0:1775767835.271284:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767835.271287:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767835.271293:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767835.271307:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742100736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767835.271311:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767835/real 1775767835] req@ffff90c203638700 x1862023742100352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767835.271343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742100352/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767835.271357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767835.271361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767835.271364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742100352:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767835.271389:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767835.271408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767835.271421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767835/real 1775767835] req@ffff90c20363aa00 x1862023742100480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767835.271434:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742100480/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767835.271443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767835.271455:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767835.271469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767835.271476:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767835.271495:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767835.271503:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767835/real 1775767835] req@ffff90c203639180 x1862023742100608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767835.271515:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742100608/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767835.271523:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767835.271526:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767835.271529:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767835.271536:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100736:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767835.271557:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767835.271569:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767835/real 1775767835] req@ffff90c203638000 x1862023742100736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767835.271577:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742100736/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767835.271585:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767835.271587:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767835.271589:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100736:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767836.295053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767837.319027:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767838.343010:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767839.367005:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767840.390981:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767840.390990:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767840.390995:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767840.391003:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3728 00000100:00080000:0.0:1775767840.391014:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767840.391020:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767840.391036:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767840.391055:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742100864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767840.391076:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767840.391079:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767840.391081:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767840.391083:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3728 00000100:00080000:0.0:1775767840.391086:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767840.391089:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767840.391097:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767840.391106:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742100992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767840.391115:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767840.391119:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767840.391119:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742100864:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767840.391121:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767840.391127:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3728 00000100:00080000:0.0:1775767840.391130:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767840.391132:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767840.391136:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767840.391146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742101120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767840.391158:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767840.391160:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767840.391162:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767840.391164:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3728 00000100:00080000:0.0:1775767840.391166:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767840.391169:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767840.391172:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767840.391179:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767840.391182:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742101248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767840.391222:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767840/real 1775767840] req@ffff90c203638e00 x1862023742100864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767840.391237:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742100864/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767840.391245:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767840.391248:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767840.391251:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742100864:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767840.391269:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100992:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767840.391284:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767840.391292:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767840/real 1775767840] req@ffff90c203638380 x1862023742100992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767840.391312:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742100992/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767840.391321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767840.391324:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767840.391328:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742100992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767840.391334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767840.391364:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767840.391377:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767840/real 1775767840] req@ffff90c20363a680 x1862023742101120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767840.391390:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742101120/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767840.391399:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767840.391403:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767840.391407:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767840.391442:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101248:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767840.391459:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767840.391467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767840/real 1775767840] req@ffff90c20363b100 x1862023742101248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767840.391475:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742101248/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767840.391487:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767840.391489:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767840.391492:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101248:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767841.414972:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767842.439014:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767843.463104:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767844.487096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767845.511005:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767845.511016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767845.511021:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767845.511033:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3733 00000100:00080000:0.0:1775767845.511056:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767845.511065:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775767845.511109:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742101376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775767845.511125:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767845.511145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767845.511150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767845.511152:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767845.511157:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3733 00000100:00080000:0.0:1775767845.511161:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767845.511166:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767845.511178:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767845.511193:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742101504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767845.511206:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742101376:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767845.511211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767845.511221:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767845.511223:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767845.511229:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3733 00000100:00080000:0.0:1775767845.511234:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767845.511239:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767845.511245:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767845.511262:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742101632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767845.511273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767845.511277:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767845.511279:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767845.511283:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3733 00000100:00080000:0.0:1775767845.511289:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767845.511293:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767845.511298:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767845.511310:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742101760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767845.511329:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767845.511414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767845/real 1775767845] req@ffff90c203639500 x1862023742101376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767845.511438:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742101376/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767845.511451:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767845.511456:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767845.511460:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742101376:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767845.511490:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767845.511516:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767845.511529:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767845/real 1775767845] req@ffff90c20363b480 x1862023742101504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767845.511541:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742101504/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767845.511549:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767845.511552:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767845.511555:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767845.511562:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767845.511578:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767845.511586:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767845/real 1775767845] req@ffff90c203639f80 x1862023742101632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767845.511597:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742101632/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767845.511604:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767845.511606:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767845.511609:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767845.511615:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101760:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767845.511630:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767845.511637:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767845/real 1775767845] req@ffff90c20363bb80 x1862023742101760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767845.511647:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742101760/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767845.511657:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767845.511660:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767845.511663:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742101760:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767846.535091:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767847.559025:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767848.583044:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767849.607032:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767850.631021:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767850.631035:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767850.631042:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767850.631054:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3739 00000100:00080000:0.0:1775767850.631068:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767850.631076:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767850.631098:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767850.631126:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742101888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767850.631152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767850.631156:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767850.631157:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767850.631160:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3739 00000100:00080000:0.0:1775767850.631163:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767850.631165:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767850.631174:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767850.631188:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742102016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767850.631196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767850.631201:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767850.631202:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742101888:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767850.631203:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767850.631208:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3739 00000100:00080000:0.0:1775767850.631213:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767850.631215:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767850.631218:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767850.631226:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742102144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767850.631238:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767850.631241:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767850.631243:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767850.631246:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3739 00000100:00080000:0.0:1775767850.631247:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767850.631250:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767850.631253:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767850.631261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742102272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767850.631302:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767850.631363:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767850/real 1775767850] req@ffff90c20363b800 x1862023742101888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767850.631381:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742101888/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767850.631390:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767850.631393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767850.631397:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742101888:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767850.631419:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767850.631437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767850.631445:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767850/real 1775767850] req@ffff90c20363bb80 x1862023742102016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767850.631456:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742102016/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767850.631465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767850.631469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767850.631481:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767850.631489:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767850.631510:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767850.631522:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767850/real 1775767850] req@ffff90c203639f80 x1862023742102144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767850.631534:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742102144/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767850.631549:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767850.631552:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767850.631556:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767850.631562:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102272:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767850.631582:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767850.631593:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767850/real 1775767850] req@ffff90c20363b480 x1862023742102272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767850.631601:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742102272/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767850.631609:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767850.631611:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767850.631613:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102272:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767851.655023:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767852.679060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767853.703012:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767854.727045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767855.750968:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767855.750976:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767855.750981:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767855.750990:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3744 00000100:00080000:1.0:1775767855.751000:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775767855.751000:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767855.751008:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767855.751040:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742102400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767855.751061:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767855.751064:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767855.751065:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767855.751068:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3744 00000100:00080000:1.0:1775767855.751070:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767855.751073:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767855.751080:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767855.751091:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742102528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767855.751096:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742102400:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767855.751104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767855.751107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767855.751109:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767855.751112:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3744 00000100:00080000:1.0:1775767855.751115:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767855.751117:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767855.751120:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767855.751128:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742102656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767855.751135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767855.751138:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767855.751139:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767855.751141:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3744 00000100:00080000:1.0:1775767855.751144:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767855.751147:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767855.751150:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767855.751160:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742102784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767855.751160:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767855.751195:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767855/real 1775767855] req@ffff90c203497480 x1862023742102400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767855.751221:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742102400/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767855.751229:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767855.751232:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767855.751235:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742102400:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767855.751257:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102528:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767855.751293:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767855.751307:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767855/real 1775767855] req@ffff90c203494000 x1862023742102528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767855.751321:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742102528/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767855.751330:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767855.751334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767855.751336:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767855.751341:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767855.751356:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767855.751364:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767855/real 1775767855] req@ffff90c203494380 x1862023742102656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767855.751371:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742102656/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767855.751376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767855.751378:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767855.751380:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767855.751384:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102784:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767855.751394:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767855.751398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767855/real 1775767855] req@ffff90c203495f80 x1862023742102784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767855.751406:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742102784/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767855.751412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767855.751414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767855.751416:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742102784:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767856.775059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767857.799006:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767858.823117:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767859.846981:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767860.871018:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767860.871029:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775767860.871034:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767860.871044:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3749 00000100:00080000:2.0:1775767860.871055:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775767860.871061:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775767860.871071:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767860.871109:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bfb80 x1862023742102912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767860.871140:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767860.871143:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775767860.871145:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767860.871148:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3749 00000100:00080000:2.0:1775767860.871151:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767860.871153:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767860.871162:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767860.871176:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd500 x1862023742103040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767860.871186:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742102912:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775767860.871205:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775767860.871208:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775767860.871223:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767860.871229:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3749 00000100:00080000:2.0:1775767860.871234:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767860.871238:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767860.871244:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767860.871257:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742103168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775767860.871265:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1775767860.871270:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775767860.871271:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775767860.871274:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775767860.871278:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3749 00000100:00080000:2.0:1775767860.871282:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775767860.871286:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775767860.871290:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767860.871311:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767860/real 1775767860] req@ffff90c2034bfb80 x1862023742102912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767860.871311:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742103296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775767860.871346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bfb80 x1862023742102912/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767860.871355:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767860.871367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767860.871371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742102912:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767860.871393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767860.871418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767860.871441:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767860/real 1775767860] req@ffff90c2034bd500 x1862023742103040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767860.871453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd500 x1862023742103040/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767860.871459:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767860.871461:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767860.871464:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767860.871468:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767860.871483:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767860.871490:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767860/real 1775767860] req@ffff90c2034bea00 x1862023742103168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767860.871497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742103168/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767860.871502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767860.871504:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767860.871506:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767860.871509:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103296:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767860.871519:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767860.871524:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767860/real 1775767860] req@ffff90c2034bca80 x1862023742103296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767860.871531:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742103296/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767860.871543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767860.871544:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767860.871546:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103296:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767861.895055:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767862.919042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767863.943036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767864.967052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767865.990995:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767865.991003:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767865.991007:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767865.991022:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3754 00000100:00080000:1.0:1775767865.991034:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767865.991041:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767865.991073:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767865.991089:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742103424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767865.991116:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767865.991120:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767865.991122:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767865.991126:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3754 00000100:00080000:1.0:1775767865.991130:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767865.991134:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767865.991143:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767865.991178:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742103552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767865.991187:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767865.991190:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767865.991191:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767865.991195:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3754 00000100:00080000:1.0:1775767865.991198:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767865.991201:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767865.991205:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767865.991210:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742103424:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767865.991216:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742103680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767865.991224:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767865.991226:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767865.991227:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767865.991230:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3754 00000100:00080000:1.0:1775767865.991233:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767865.991236:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767865.991240:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767865.991248:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742103808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775767865.991269:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767865.991302:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffff90c203494e00 x1862023742103424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767865.991316:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742103424/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767865.991323:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767865.991326:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767865.991329:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742103424:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775767865.991334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103552:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767865.991346:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767865.991353:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffff90c203496d80 x1862023742103552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767865.991361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742103552/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767865.991366:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767865.991367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767865.991369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767865.991388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775767865.991403:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767865.991413:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffff90c203497100 x1862023742103680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767865.991425:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742103680/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767865.991433:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767865.991436:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767865.991440:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775767865.991446:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103808:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775767865.991468:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775767865.991478:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767865/real 1775767865] req@ffff90c203495880 x1862023742103808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775767865.991497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742103808/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767890 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775767865.991507:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775767865.991509:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775767865.991511:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742103808:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767867.015040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767868.039128:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767869.063026:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767870.087142:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775767871.111017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767871.111028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775767871.111033:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767871.111045:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3759 00000100:00080000:0.0:1775767871.111056:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767871.111062:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767871.111094:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775767871.111105:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742103936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767871.111162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767871.111165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775767871.111166:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767871.111170:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3759 00000100:00080000:0.0:1775767871.111173:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767871.111177:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767871.111187:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767871.111202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742104064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767871.111214:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742103936:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775767871.111216:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767871.111236:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775767871.111237:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767871.111241:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3759 00000100:00080000:0.0:1775767871.111243:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767871.111246:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767871.111251:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775767871.111261:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742104192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775767871.111273:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775767871.111276:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775767871.111284:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775767871.111286:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3759 00000100:00080000:0.0:1775767871.111289:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775767871.111292:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775767871.111296:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775767871.111308:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742104320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767871.111318:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767871.111370:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767871/real 1775767871] req@ffff90c203639500 x1862023742103936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767871.111392:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742103936/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767871.111401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767871.111405:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767871.111409:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742103936:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767871.111435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767871.111459:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767871.111473:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767871/real 1775767871] req@ffff90c20363b100 x1862023742104064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767871.111500:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742104064/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767871.111507:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767871.111509:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767871.111512:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767871.111521:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767871.111539:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767871.111546:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767871/real 1775767871] req@ffff90c20363a680 x1862023742104192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767871.111554:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742104192/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767871.111566:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767871.111570:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767871.111574:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767871.111581:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104320:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767871.111603:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767871.111614:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767871/real 1775767871] req@ffff90c203638380 x1862023742104320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767871.111626:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742104320/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767871.111635:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767871.111638:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767871.111640:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104320:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767872.135139:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767873.159064:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767874.183158:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767875.207058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767876.231012:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767876.231024:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767876.231028:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767876.231041:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3764 00000100:00080000:1.0:1775767876.231053:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767876.231078:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767876.231092:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767876.231115:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742104448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767876.231140:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767876.231143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767876.231145:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767876.231148:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3764 00000100:00080000:1.0:1775767876.231151:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767876.231153:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767876.231162:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767876.231171:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742104576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767876.231180:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767876.231188:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767876.231189:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742104448:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767876.231192:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767876.231216:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3764 00000100:00080000:1.0:1775767876.231226:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767876.231231:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767876.231237:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767876.231252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742104704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767876.231260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767876.231263:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767876.231264:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767876.231267:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3764 00000100:00080000:1.0:1775767876.231271:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767876.231277:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767876.231279:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775767876.231288:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767876.231298:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742104832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767876.231343:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767876/real 1775767876] req@ffff90c203495c00 x1862023742104448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767876.231366:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742104448/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767876.231375:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767876.231379:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767876.231382:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742104448:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767876.231409:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767876.231428:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767876.231438:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767876/real 1775767876] req@ffff90c203497b80 x1862023742104576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767876.231446:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742104576/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767876.231452:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767876.231454:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767876.231456:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767876.231460:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104704:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767876.231471:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767876.231477:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767876/real 1775767876] req@ffff90c203496300 x1862023742104704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767876.231484:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742104704/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767876.231489:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767876.231491:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767876.231492:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767876.231496:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104832:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767876.231506:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767876.231511:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767876/real 1775767876] req@ffff90c203496a00 x1862023742104832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767876.231519:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742104832/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767876.231526:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767876.231528:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767876.231530:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742104832:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767877.255037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767878.279182:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767879.303111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767880.327127:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767881.351063:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767881.351074:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767881.351080:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767881.351091:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3769 00010000:00010000:1.0:1775767881.351105:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767881.351113:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767881.351122:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767881.351162:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742104960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767881.351193:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767881.351200:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767881.351202:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767881.351207:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3769 00000100:00080000:3.0:1775767881.351214:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767881.351227:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767881.351238:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767881.351243:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742104960:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767881.351256:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742105088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767881.351271:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767881.351275:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767881.351277:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767881.351282:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3769 00000100:00080000:3.0:1775767881.351287:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767881.351291:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767881.351298:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767881.351309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742105216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767881.351320:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767881.351324:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1775767881.351327:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767881.351329:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767881.351337:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3769 00000100:00080000:3.0:1775767881.351345:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767881.351350:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767881.351359:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767881.351375:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767881/real 1775767881] req@ffff90c218da3800 x1862023742104960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775767881.351381:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742105344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767881.351413:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742104960/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767881.351425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767881.351429:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767881.351434:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742104960:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767881.351461:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105088:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767881.351477:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767881.351485:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767881/real 1775767881] req@ffff90c218da2d80 x1862023742105088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767881.351492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742105088/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767881.351503:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767881.351506:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767881.351508:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767881.351512:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105216:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767881.351522:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767881.351528:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767881/real 1775767881] req@ffff90c218da0e00 x1862023742105216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767881.351535:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742105216/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767881.351540:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767881.351542:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767881.351544:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105216:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767881.351547:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105344:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767881.351558:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767881.351563:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767881/real 1775767881] req@ffff90c218da1880 x1862023742105344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767881.351569:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742105344/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767881.351577:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767881.351578:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767881.351581:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105344:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767882.375115:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767883.399056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767884.423007:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767885.447027:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767886.470973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767886.470984:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767886.470989:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767886.470999:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3774 00000100:00080000:3.0:1775767886.471010:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767886.471017:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767886.471029:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767886.471057:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742105472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767886.471087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767886.471091:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767886.471094:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767886.471102:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3774 00000100:00100000:2.0:1775767886.471104:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742105472:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767886.471107:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767886.471114:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767886.471124:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767886.471140:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742105600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767886.471149:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767886.471153:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767886.471154:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767886.471157:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3774 00000100:00080000:3.0:1775767886.471161:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767886.471163:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767886.471168:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775767886.471169:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767886.471179:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742105728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767886.471187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767886.471190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767886.471191:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767886.471193:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3774 00000100:00080000:3.0:1775767886.471195:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767886.471198:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767886.471201:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767886.471214:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742105856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767886.471232:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767886/real 1775767886] req@ffff90c218da0a80 x1862023742105472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767886.471249:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742105472/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767886.471257:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767886.471261:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767886.471265:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742105472:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767886.471285:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767886.471302:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767886.471313:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767886/real 1775767886] req@ffff90c218da0700 x1862023742105600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767886.471321:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742105600/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767886.471327:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767886.471329:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767886.471332:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767886.471338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105728:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767886.471356:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767886.471367:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767886/real 1775767886] req@ffff90c218da0380 x1862023742105728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767886.471380:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742105728/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767886.471390:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767886.471393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767886.471396:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105728:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767886.471403:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105856:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767886.471425:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767886.471436:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767886/real 1775767886] req@ffff90c218da3480 x1862023742105856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767886.471448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742105856/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767886.471459:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767886.471463:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767886.471466:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742105856:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767887.495076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767888.519084:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767889.543099:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767890.567019:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767891.590987:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767891.590995:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767891.590999:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767891.591008:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3780 00000100:00080000:3.0:1775767891.591018:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767891.591025:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767891.591049:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767891.591060:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742105984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767891.591080:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767891.591084:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767891.591085:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767891.591088:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3780 00000100:00080000:3.0:1775767891.591091:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767891.591093:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767891.591101:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767891.591110:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742106112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767891.591114:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742105984:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767891.591127:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767891.591130:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767891.591131:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767891.591134:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3780 00000100:00080000:3.0:1775767891.591136:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767891.591138:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767891.591141:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767891.591150:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742106240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767891.591156:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767891.591158:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767891.591159:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767891.591162:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3780 00000100:00080000:3.0:1775767891.591171:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767891.591174:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767891.591178:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767891.591187:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742106368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767891.591224:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767891.591253:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767891.591268:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106368:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767892.615157:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767893.639062:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767894.663010:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775767894.663109:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775767894.663147:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775767894.663164:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767894.663203:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767894.663214:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775767894.663222:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767894.663252:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767891/real 1775767894] req@ffff90c218da0000 x1862023742106112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767894.663280:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742106112/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767894.663292:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767894.663297:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767894.663302:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767894.663311:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767891/real 1775767894] req@ffff90c218da1f80 x1862023742106240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767894.663324:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742106240/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767894.663332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767894.663336:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767894.663339:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106240:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767894.663347:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767891/real 1775767894] req@ffff90c218da3480 x1862023742106368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767894.663359:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742106368/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767894.663370:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767894.663374:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767894.663378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106368:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775767894.663386:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767891/real 1775767894] req@ffff90c218da3100 x1862023742105984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767894.663401:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742105984/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767894.663410:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767894.663414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767894.663417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742105984:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775767895.687100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767896.710962:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767896.710974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767896.710979:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767896.710991:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3785 00000100:00080000:3.0:1775767896.711004:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775767896.711013:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767896.711013:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767896.711065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742106496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767896.711098:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767896.711101:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767896.711103:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767896.711108:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3785 00000100:00080000:3.0:1775767896.711113:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767896.711119:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767896.711132:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767896.711145:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742106496:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767896.711155:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742106624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767896.711173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767896.711177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767896.711179:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767896.711184:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3785 00000100:00080000:3.0:1775767896.711188:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767896.711193:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767896.711199:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767896.711215:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742106752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767896.711223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767896.711225:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767896.711227:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767896.711230:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3785 00000800:00000100:2.0:1775767896.711239:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767896.711240:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767896.711248:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767896.711253:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767896.711274:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742106880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767896.711298:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767896/real 1775767896] req@ffff90c218da0380 x1862023742106496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767896.711322:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742106496/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767896.711336:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767896.711342:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767896.711347:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742106496:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767896.711377:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767896.711401:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767896.711414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767896/real 1775767896] req@ffff90c218da0700 x1862023742106624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767896.711427:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742106624/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767896.711437:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767896.711441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767896.711445:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767896.711452:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106752:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767896.711474:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767896.711484:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767896/real 1775767896] req@ffff90c218da0a80 x1862023742106752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767896.711497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742106752/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767896.711506:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767896.711510:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767896.711514:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106752:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767896.711520:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106880:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767896.711542:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767896.711564:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767896/real 1775767896] req@ffff90c218da1880 x1862023742106880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767896.711577:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742106880/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767896.711587:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767896.711591:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767896.711595:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742106880:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767897.735039:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767898.759047:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767899.783044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767900.806998:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767901.830957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767901.830966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767901.830971:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767901.830983:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3790 00000100:00080000:3.0:1775767901.830996:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767901.831004:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767901.831013:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767901.831044:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742107008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767901.831067:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767901.831070:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767901.831071:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767901.831074:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3790 00000100:00080000:3.0:1775767901.831077:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767901.831080:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767901.831089:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767901.831102:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742107136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767901.831109:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742107008:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767901.831114:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767901.831117:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767901.831119:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767901.831122:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3790 00000100:00080000:3.0:1775767901.831125:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767901.831127:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767901.831130:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767901.831137:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742107264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767901.831144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767901.831147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767901.831148:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767901.831151:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3790 00000100:00080000:3.0:1775767901.831165:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767901.831167:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767901.831170:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767901.831176:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767901.831180:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742107392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767901.831217:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767901/real 1775767901] req@ffff90c218da0e00 x1862023742107008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767901.831238:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742107008/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767901.831248:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767901.831254:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767901.831258:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742107008:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767901.831282:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767901.831307:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767901.831319:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767901/real 1775767901] req@ffff90c218da2d80 x1862023742107136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767901.831346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742107136/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767901.831357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767901.831360:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767901.831364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767901.831370:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107264:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767901.831391:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767901.831403:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767901/real 1775767901] req@ffff90c218da3800 x1862023742107264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767901.831416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742107264/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767901.831425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767901.831428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767901.831432:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107264:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767901.831438:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107392:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767901.831457:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767901.831467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767901/real 1775767901] req@ffff90c218da1880 x1862023742107392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767901.831480:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742107392/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767901.831491:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767901.831494:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767901.831497:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107392:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767902.855024:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767903.879043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767904.903074:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767905.927001:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767906.950975:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767906.950983:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767906.950987:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767906.950995:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3795 00000100:00080000:3.0:1775767906.951006:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767906.951012:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767906.951047:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742107520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775767906.951057:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767906.951069:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767906.951072:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767906.951074:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767906.951077:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3795 00000100:00080000:3.0:1775767906.951080:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767906.951084:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767906.951096:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767906.951098:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742107520:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767906.951112:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742107648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767906.951126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767906.951130:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767906.951132:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767906.951137:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3795 00000100:00080000:3.0:1775767906.951144:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767906.951148:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767906.951153:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775767906.951156:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767906.951169:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742107776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767906.951177:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767906.951180:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767906.951182:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775767906.951186:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767906/real 1775767906] req@ffff90c218da0a80 x1862023742107520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767906.951199:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3795 00000100:00100000:2.0:1775767906.951203:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742107520/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767906.951205:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767906.951212:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775767906.951217:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1775767906.951219:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775767906.951223:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767906.951227:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742107520:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767906.951233:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742107904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767906.951249:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107648:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767906.951266:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767906.951273:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767906/real 1775767906] req@ffff90c218da0700 x1862023742107648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767906.951281:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742107648/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767906.951287:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767906.951289:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767906.951291:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767906.951295:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107776:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767906.951304:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767906.951309:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767906/real 1775767906] req@ffff90c218da0380 x1862023742107776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767906.951316:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742107776/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767906.951321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767906.951323:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767906.951325:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107776:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767906.951330:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107904:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767906.951340:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767906.951345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767906/real 1775767906] req@ffff90c218da3100 x1862023742107904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767906.951353:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742107904/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767931 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767906.951359:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767906.951361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767906.951363:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742107904:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767907.975022:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767908.999069:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767910.023046:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767911.047034:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767912.070968:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767912.070977:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767912.070981:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767912.070990:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3800 00000100:00080000:3.0:1775767912.071000:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767912.071006:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767912.071028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767912.071043:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742108032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767912.071071:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767912.071075:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767912.071077:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767912.071082:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3800 00000100:00080000:3.0:1775767912.071087:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767912.071092:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767912.071103:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767912.071112:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742108032:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767912.071117:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742108160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767912.071128:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767912.071131:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767912.071133:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767912.071136:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3800 00000100:00080000:3.0:1775767912.071139:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767912.071142:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767912.071145:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767912.071153:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742108288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767912.071161:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767912.071163:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767912.071164:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767912.071167:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3800 00000100:00080000:3.0:1775767912.071170:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767912.071172:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767912.071175:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767912.071182:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767912.071184:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742108416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767912.071224:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767912/real 1775767912] req@ffff90c218da3480 x1862023742108032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767912.071242:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742108032/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767912.071253:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767912.071258:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767912.071262:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742108032:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767912.071288:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767912.071324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767912.071338:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767912/real 1775767912] req@ffff90c218da1f80 x1862023742108160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767912.071348:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742108160/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767912.071353:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767912.071356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767912.071358:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767912.071362:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108288:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767912.071376:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767912.071382:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767912/real 1775767912] req@ffff90c218da0000 x1862023742108288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767912.071390:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742108288/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767912.071394:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767912.071396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767912.071398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108288:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767912.071401:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108416:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767912.071411:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767912.071416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767912/real 1775767912] req@ffff90c218da3100 x1862023742108416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767912.071423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742108416/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767912.071434:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767912.071436:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767912.071438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108416:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767913.095056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767914.119003:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767915.143096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767916.167015:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767917.190983:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767917.190991:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767917.190995:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767917.191004:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3805 00000100:00080000:1.0:1775767917.191015:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767917.191022:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767917.191039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767917.191056:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742108544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767917.191078:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767917.191081:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767917.191082:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767917.191084:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3805 00000100:00080000:1.0:1775767917.191087:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767917.191091:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767917.191100:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767917.191110:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742108544:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767917.191112:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742108672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767917.191130:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767917.191132:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767917.191134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767917.191137:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3805 00000100:00080000:1.0:1775767917.191139:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767917.191142:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767917.191145:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767917.191153:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742108800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767917.191160:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767917.191163:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767917.191166:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767917.191167:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767917.191171:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3805 00000100:00080000:1.0:1775767917.191174:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767917.191176:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767917.191180:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767917.191188:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742108928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767917.191210:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767917/real 1775767917] req@ffff90c203496680 x1862023742108544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767917.191232:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742108544/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767917.191242:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767917.191246:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767917.191249:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742108544:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767917.191268:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767917.191286:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767917.191294:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767917/real 1775767917] req@ffff90c203494a80 x1862023742108672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767917.191303:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742108672/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767917.191309:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767917.191311:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767917.191313:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767917.191318:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108800:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767917.191328:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767917.191334:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767917/real 1775767917] req@ffff90c203494700 x1862023742108800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767917.191342:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742108800/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767917.191347:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767917.191349:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767917.191351:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108800:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767917.191354:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108928:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767917.191371:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767917.191382:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767917/real 1775767917] req@ffff90c203495500 x1862023742108928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767917.191393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742108928/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767917.191403:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767917.191406:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767917.191410:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742108928:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767918.215059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767919.239086:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767920.263085:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767921.287092:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767922.311003:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767922.311013:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767922.311018:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767922.311029:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3810 00000100:00080000:3.0:1775767922.311042:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775767922.311050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767922.311050:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767922.311100:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742109056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767922.311126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767922.311130:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767922.311131:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767922.311134:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3810 00000100:00080000:3.0:1775767922.311136:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767922.311138:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767922.311147:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767922.311156:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742109184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767922.311163:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767922.311166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767922.311167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767922.311171:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3810 00000100:00080000:3.0:1775767922.311172:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767922.311175:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767922.311178:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767922.311184:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742109056:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767922.311188:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742109312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767922.311197:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767922.311199:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767922.311200:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767922.311203:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3810 00000100:00080000:3.0:1775767922.311205:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767922.311207:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767922.311212:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767922.311222:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742109440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767922.311262:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767922.311312:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767922/real 1775767922] req@ffff90c218da0380 x1862023742109056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767922.311333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742109056/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767922.311343:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767922.311348:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767922.311351:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742109056:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767922.311357:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109184:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767922.311375:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767922.311384:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767922/real 1775767922] req@ffff90c218da0700 x1862023742109184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767922.311392:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742109184/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767922.311400:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767922.311404:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767922.311407:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767922.311439:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109312:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767922.311460:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767922.311470:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767922/real 1775767922] req@ffff90c218da0a80 x1862023742109312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767922.311478:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742109312/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767922.311491:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767922.311495:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767922.311498:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109312:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767922.311504:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109440:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767922.311525:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767922.311536:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767922/real 1775767922] req@ffff90c218da1880 x1862023742109440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767922.311548:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742109440/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767922.311556:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767922.311559:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767922.311561:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109440:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767923.335018:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767924.359051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767925.383102:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767926.407056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767927.430967:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767927.430977:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767927.430982:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767927.430992:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3815 00000100:00080000:3.0:1775767927.431004:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767927.431011:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767927.431042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767927.431057:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742109568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767927.431079:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767927.431083:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767927.431085:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767927.431089:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3815 00000100:00080000:3.0:1775767927.431094:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767927.431097:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767927.431106:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767927.431112:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742109568:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767927.431117:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742109696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767927.431127:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767927.431131:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767927.431133:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767927.431136:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3815 00000100:00080000:3.0:1775767927.431139:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767927.431143:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767927.431147:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767927.431155:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742109824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767927.431163:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767927.431166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767927.431168:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767927.431169:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775767927.431172:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3815 00000100:00080000:3.0:1775767927.431175:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767927.431178:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767927.431182:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767927.431192:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742109952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767927.431204:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767927/real 1775767927] req@ffff90c218da3800 x1862023742109568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767927.431222:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742109568/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767927.431232:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767927.431236:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767927.431240:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742109568:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767927.431260:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767927.431277:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767927.431285:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767927/real 1775767927] req@ffff90c218da2d80 x1862023742109696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767927.431294:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742109696/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767927.431299:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767927.431301:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767927.431303:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767927.431307:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109824:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767927.431316:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767927.431321:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767927/real 1775767927] req@ffff90c218da0e00 x1862023742109824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767927.431328:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742109824/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767927.431332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767927.431334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767927.431336:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109824:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767927.431339:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109952:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767927.431348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767927.431353:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767927/real 1775767927] req@ffff90c218da1880 x1862023742109952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767927.431360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742109952/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767927.431366:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767927.431368:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767927.431369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742109952:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767928.454995:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767929.478988:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767930.503050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767931.527026:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767932.550979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767932.550987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767932.550991:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767932.551000:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3821 00000100:00080000:1.0:1775767932.551010:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767932.551016:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767932.551021:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767932.551048:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742110080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767932.551069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767932.551072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767932.551073:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767932.551076:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3821 00000100:00080000:1.0:1775767932.551078:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767932.551081:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767932.551108:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767932.551115:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742110080:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775767932.551121:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742110208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767932.551130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767932.551133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767932.551134:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767932.551138:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3821 00000100:00080000:1.0:1775767932.551140:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767932.551143:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767932.551147:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767932.551155:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742110336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767932.551161:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767932.551163:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767932.551165:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767932.551168:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3821 00000100:00080000:1.0:1775767932.551170:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767932.551172:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767932.551175:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767932.551180:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767932.551186:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742110464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767932.551215:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767932/real 1775767932] req@ffff90c203495180 x1862023742110080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767932.551241:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742110080/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767932.551249:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767932.551252:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767932.551255:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742110080:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767932.551272:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110208:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767932.551287:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767932.551294:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767932/real 1775767932] req@ffff90c203495500 x1862023742110208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767932.551302:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742110208/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767932.551312:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767932.551314:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767932.551316:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767932.551319:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110336:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767932.551329:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767932.551334:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767932/real 1775767932] req@ffff90c203494700 x1862023742110336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767932.551340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742110336/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767932.551345:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767932.551347:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767932.551349:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110336:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767932.551352:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110464:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767932.551361:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767932.551366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767932/real 1775767932] req@ffff90c203494a80 x1862023742110464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767932.551373:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742110464/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767932.551379:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767932.551381:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767932.551384:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110464:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767933.575004:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767934.599045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767935.622990:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767936.647050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767937.670994:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767937.671008:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767937.671016:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767937.671029:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3826 00000100:00080000:1.0:1775767937.671045:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767937.671054:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767937.671065:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767937.671110:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742110592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767937.671147:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767937.671150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767937.671152:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767937.671155:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3826 00000100:00080000:1.0:1775767937.671159:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767937.671164:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767937.671177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767937.671192:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742110720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767937.671194:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742110592:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767937.671220:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767937.671223:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767937.671224:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767937.671229:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3826 00000100:00080000:1.0:1775767937.671232:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767937.671235:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767937.671239:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767937.671251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742110848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767937.671257:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767937.671259:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767937.671260:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767937.671263:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3826 00000100:00080000:1.0:1775767937.671267:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767937.671270:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767937.671274:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767937.671277:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775767937.671290:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742110976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767937.671325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767937/real 1775767937] req@ffff90c203496680 x1862023742110592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767937.671341:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742110592/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767937.671349:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767937.671352:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767937.671355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742110592:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767937.671375:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767937.671390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767937.671397:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767937/real 1775767937] req@ffff90c203496a00 x1862023742110720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767937.671406:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742110720/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767937.671411:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767937.671413:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767937.671416:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767937.671419:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110848:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767937.671429:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767937.671435:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767937/real 1775767937] req@ffff90c203496300 x1862023742110848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767937.671443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742110848/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767937.671448:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767937.671450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767937.671452:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110848:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767937.671455:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110976:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767937.671465:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767937.671487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767937/real 1775767937] req@ffff90c203497b80 x1862023742110976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767937.671495:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742110976/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767937.671502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767937.671504:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767937.671506:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742110976:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767938.695093:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767939.719150:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767940.743090:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767941.767101:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767942.790998:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767942.791010:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767942.791017:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:1.0:1775767942.791031:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767942.791031:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3831 00000100:00080000:3.0:1775767942.791047:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767942.791055:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767942.791105:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742111104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767942.791138:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767942.791143:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767942.791145:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767942.791150:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3831 00000100:00080000:3.0:1775767942.791154:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767942.791158:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767942.791169:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767942.791185:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742111232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767942.791197:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742111104:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767942.791199:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767942.791207:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767942.791209:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767942.791214:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3831 00000100:00080000:3.0:1775767942.791218:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767942.791222:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767942.791229:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767942.791242:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742111360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767942.791253:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767942.791256:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767942.791258:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767942.791263:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3831 00000100:00080000:3.0:1775767942.791267:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767942.791272:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767942.791276:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775767942.791280:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767942.791296:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742111488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767942.791320:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767942/real 1775767942] req@ffff90c218da0a80 x1862023742111104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767942.791336:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742111104/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767942.791346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767942.791349:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767942.791353:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742111104:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767942.791373:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767942.791386:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767942.791394:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767942/real 1775767942] req@ffff90c218da0700 x1862023742111232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767942.791402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742111232/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767942.791411:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767942.791415:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767942.791450:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767942.791457:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111360:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767942.791479:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767942.791489:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767942/real 1775767942] req@ffff90c218da0380 x1862023742111360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767942.791497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742111360/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767942.791503:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767942.791505:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767942.791507:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111360:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767942.791512:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111488:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767942.791525:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767942.791530:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767942/real 1775767942] req@ffff90c218da3100 x1862023742111488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767942.791538:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742111488/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767942.791545:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767942.791547:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767942.791548:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111488:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767943.815048:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767944.839028:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767945.863097:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767946.887002:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767947.911017:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767947.911028:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767947.911033:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767947.911044:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3836 00000100:00080000:3.0:1775767947.911057:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767947.911065:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767947.911096:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767947.911112:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742111616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767947.911138:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767947.911141:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767947.911143:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767947.911146:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3836 00000100:00080000:3.0:1775767947.911148:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767947.911150:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767947.911160:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767947.911170:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742111744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767947.911178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767947.911180:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767947.911181:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767947.911184:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3836 00000100:00080000:3.0:1775767947.911187:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767947.911192:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742111616:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767947.911193:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767947.911198:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767947.911216:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742111872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767947.911224:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767947.911226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767947.911228:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767947.911232:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3836 00000100:00080000:3.0:1775767947.911234:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767947.911236:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767947.911241:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767947.911252:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742112000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767947.911277:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767947.911310:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767947/real 1775767947] req@ffff90c218da0000 x1862023742111616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767947.911324:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742111616/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767947.911333:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767947.911337:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767947.911339:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742111616:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767947.911360:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111744:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767947.911391:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767947.911400:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767947/real 1775767947] req@ffff90c218da1f80 x1862023742111744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767947.911409:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742111744/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767947.911415:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767947.911417:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767947.911420:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767947.911424:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111872:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767947.911436:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767947.911441:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767947/real 1775767947] req@ffff90c218da3480 x1862023742111872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767947.911449:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742111872/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767947.911453:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767947.911455:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767947.911457:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742111872:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767947.911461:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112000:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767947.911471:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767947.911476:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767947/real 1775767947] req@ffff90c218da3100 x1862023742112000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767947.911482:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742112000/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767947.911493:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767947.911495:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767947.911497:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112000:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767948.935011:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767949.959056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767950.983030:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767952.007082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767953.030990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767953.031005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00010000:00010000:3.0:1775767953.031007:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767953.031012:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767953.031024:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3841 00000100:00080000:1.0:1775767953.031036:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767953.031043:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767953.031091:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742112128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767953.031118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767953.031121:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767953.031123:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767953.031126:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3841 00000100:00080000:1.0:1775767953.031128:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767953.031131:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767953.031142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767953.031154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742112256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767953.031154:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742112128:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767953.031169:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767953.031171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767953.031172:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767953.031175:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3841 00000100:00080000:1.0:1775767953.031177:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767953.031179:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767953.031182:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767953.031190:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742112384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767953.031196:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767953.031198:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767953.031199:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767953.031202:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3841 00000100:00080000:1.0:1775767953.031205:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767953.031207:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767953.031211:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767953.031217:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742112512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767953.031231:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767953.031270:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767953/real 1775767953] req@ffff90c203495c00 x1862023742112128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767953.031285:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742112128/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767953.031294:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767953.031297:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767953.031300:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742112128:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767953.031349:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767953.031366:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767953.031376:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767953/real 1775767953] req@ffff90c203495880 x1862023742112256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767953.031384:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742112256/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767953.031390:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767953.031393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767953.031395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767953.031399:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112384:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767953.031410:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767953.031416:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767953/real 1775767953] req@ffff90c203497100 x1862023742112384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767953.031423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742112384/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767953.031428:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767953.031430:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767953.031431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112384:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767953.031435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112512:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767953.031445:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767953.031450:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767953/real 1775767953] req@ffff90c203496d80 x1862023742112512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767953.031457:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742112512/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767953.031465:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767953.031467:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767953.031468:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112512:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767954.055052:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767955.079082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767956.103078:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767957.127165:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767958.151003:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:1.0:1775767958.151007:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767958.151013:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767958.151019:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767958.151029:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3846 00000100:00080000:3.0:1775767958.151040:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767958.151046:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767958.151080:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742112640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767958.151108:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767958.151112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767958.151114:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767958.151119:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3846 00000100:00080000:3.0:1775767958.151124:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767958.151129:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767958.151138:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742112640:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775767958.151139:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767958.151157:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742112768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767958.151171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767958.151173:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767958.151174:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767958.151178:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3846 00000100:00080000:3.0:1775767958.151190:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767958.151192:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767958.151196:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767958.151204:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742112896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767958.151211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767958.151213:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767958.151214:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767958.151217:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3846 00000100:00080000:3.0:1775767958.151220:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767958.151222:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767958.151226:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775767958.151234:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:3.0:1775767958.151235:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742113024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767958.151292:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767958.151329:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113024:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767959.175101:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767960.199085:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767961.223079:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775767961.223276:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775767961.223337:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775767961.223365:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767961.223414:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767961.223434:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775767961.223447:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775767961.223463:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767958/real 1775767961] req@ffff90c218da0700 x1862023742112768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767961.223497:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742112768/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767961.223513:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767961.223519:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767961.223531:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767961.223542:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767958/real 1775767961] req@ffff90c218da0a80 x1862023742112896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767961.223556:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742112896/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767961.223565:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767961.223569:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767961.223573:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742112896:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767961.223580:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767958/real 1775767961] req@ffff90c218da0380 x1862023742112640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767961.223593:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742112640/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767961.223607:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767961.223611:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767961.223614:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742112640:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767961.223647:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767958/real 1775767961] req@ffff90c218da1880 x1862023742113024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767961.223659:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742113024/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767961.223671:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767961.223675:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767961.223679:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113024:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767962.247050:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767963.270983:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767963.270993:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767963.270998:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767963.271009:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3851 00000100:00080000:3.0:1775767963.271021:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775767963.271025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767963.271029:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767963.271067:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742113152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767963.271098:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767963.271102:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767963.271104:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767963.271109:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3851 00000100:00080000:3.0:1775767963.271113:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767963.271117:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767963.271126:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767963.271140:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742113280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767963.271151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767963.271155:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767963.271157:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767963.271162:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3851 00000100:00080000:3.0:1775767963.271167:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767963.271172:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767963.271177:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767963.271185:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742113152:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767963.271191:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742113408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767963.271204:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767963.271208:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767963.271210:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767963.271215:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3851 00000100:00080000:3.0:1775767963.271234:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767963.271239:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767963.271245:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767963.271260:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742113536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767963.271285:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767963.271325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767963/real 1775767963] req@ffff90c218da0e00 x1862023742113152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767963.271346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742113152/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767963.271357:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767963.271362:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767963.271366:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742113152:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767963.271388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767963.271418:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767963.271431:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767963/real 1775767963] req@ffff90c218da2d80 x1862023742113280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767963.271443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742113280/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767963.271451:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767963.271455:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767963.271458:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767963.271486:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113408:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767963.271506:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767963.271517:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767963/real 1775767963] req@ffff90c218da3800 x1862023742113408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767963.271529:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742113408/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767963.271537:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767963.271540:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767963.271543:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113408:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767963.271549:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113536:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767963.271570:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767963.271581:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767963/real 1775767963] req@ffff90c218da1880 x1862023742113536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767963.271611:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742113536/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767963.271624:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767963.271627:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767963.271631:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113536:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767964.295097:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767965.319044:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767966.343053:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767967.366984:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767968.391036:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767968.391107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767968.391113:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767968.391117:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767968.391125:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3856 00000100:00080000:1.0:1775767968.391135:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767968.391140:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767968.391187:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742113664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767968.391207:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767968.391209:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767968.391211:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767968.391214:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3856 00000100:00080000:1.0:1775767968.391216:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767968.391218:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767968.391226:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767968.391240:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742113792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767968.391248:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767968.391251:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1775767968.391251:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742113664:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767968.391254:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767968.391261:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3856 00000100:00080000:1.0:1775767968.391263:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767968.391266:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767968.391269:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767968.391278:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742113920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767968.391284:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767968.391287:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767968.391288:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767968.391291:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3856 00000100:00080000:1.0:1775767968.391293:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767968.391295:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767968.391298:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767968.391304:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742114048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767968.391317:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767968.391356:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767968/real 1775767968] req@ffff90c203494e00 x1862023742113664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767968.391375:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742113664/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767968.391386:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767968.391399:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767968.391404:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742113664:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767968.391428:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767968.391449:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767968.391461:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767968/real 1775767968] req@ffff90c203495f80 x1862023742113792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767968.391473:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742113792/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767968.391496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767968.391500:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767968.391504:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767968.391510:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113920:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767968.391526:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767968.391535:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767968/real 1775767968] req@ffff90c203494380 x1862023742113920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767968.391546:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742113920/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767968.391555:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767968.391558:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767968.391561:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742113920:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767968.391566:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114048:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767968.391581:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767968.391587:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767968/real 1775767968] req@ffff90c203494000 x1862023742114048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767968.391600:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742114048/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767968.391610:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767968.391612:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767968.391616:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114048:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767969.414982:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767970.439036:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767971.463027:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767972.486988:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767973.511022:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767973.511031:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767973.511035:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767973.511045:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3861 00000100:00080000:1.0:1775767973.511057:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767973.511065:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775767973.511085:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775767973.511108:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742114176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767973.511151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767973.511155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767973.511157:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767973.511160:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3861 00000100:00080000:1.0:1775767973.511165:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767973.511170:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767973.511180:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767973.511197:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742114304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767973.511198:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742114176:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775767973.511223:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767973.511225:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767973.511226:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767973.511230:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3861 00000100:00080000:1.0:1775767973.511232:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767973.511234:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767973.511237:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767973.511249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742114432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767973.511256:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767973.511258:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767973.511259:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767973.511263:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3861 00000100:00080000:1.0:1775767973.511265:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767973.511268:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767973.511273:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767973.511280:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742114560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767973.511308:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767973.511362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767973/real 1775767973] req@ffff90c203497480 x1862023742114176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767973.511387:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742114176/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775767998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767973.511401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767973.511407:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767973.511412:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742114176:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767973.511443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114304:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767973.511469:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767973.511481:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767973/real 1775767973] req@ffff90c203494000 x1862023742114304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767973.511491:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742114304/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767973.511497:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767973.511499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767973.511502:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767973.511506:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114432:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767973.511519:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767973.511526:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767973/real 1775767973] req@ffff90c203494380 x1862023742114432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767973.511533:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742114432/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775767998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767973.511539:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767973.511541:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767973.511543:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114432:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767973.511547:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114560:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767973.511558:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767973.511563:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767973/real 1775767973] req@ffff90c203495f80 x1862023742114560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767973.511570:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742114560/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775767998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767973.511577:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767973.511579:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767973.511581:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114560:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767974.535061:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767975.559138:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767976.582997:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767977.607121:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767978.631004:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767978.631012:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767978.631018:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767978.631026:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3867 00000100:00080000:3.0:1775767978.631036:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767978.631042:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767978.631063:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767978.631090:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742114688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767978.631128:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767978.631131:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767978.631132:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767978.631136:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3867 00000100:00080000:3.0:1775767978.631139:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767978.631148:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767978.631161:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767978.631164:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742114688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767978.631175:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742114816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767978.631186:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767978.631191:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767978.631193:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767978.631198:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3867 00000100:00080000:3.0:1775767978.631216:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767978.631222:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767978.631226:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775767978.631233:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767978.631251:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742114944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767978.631276:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767978/real 1775767978] req@ffff90c218da0380 x1862023742114688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767978.631305:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767978.631308:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767978.631310:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767978.631316:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3867 00000100:00080000:3.0:1775767978.631321:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775767978.631324:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742114688/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775767978.631329:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767978.631337:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00080000:2.0:1775767978.631343:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767978.631351:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:3.0:1775767978.631353:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742115072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767978.631361:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742114688:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767978.631388:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767978.631412:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767978.631421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767978/real 1775767978] req@ffff90c218da0a80 x1862023742114816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767978.631430:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742114816/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767978.631436:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767978.631438:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767978.631440:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767978.631444:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114944:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767978.631455:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767978.631460:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767978/real 1775767978] req@ffff90c218da0700 x1862023742114944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767978.631467:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742114944/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767978.631472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767978.631474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767978.631476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742114944:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767978.631480:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115072:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767978.631491:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767978.631496:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767978/real 1775767978] req@ffff90c218da3100 x1862023742115072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767978.631503:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742115072/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767978.631512:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767978.631515:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767978.631519:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115072:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767979.655035:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767980.679115:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767981.703094:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767982.727096:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767983.751045:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767983.751053:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767983.751058:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767983.751067:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3872 00010000:00010000:1.0:1775767983.751075:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767983.751077:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767983.751089:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775767983.751124:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742115200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767983.751149:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767983.751153:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767983.751154:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767983.751160:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3872 00000100:00080000:3.0:1775767983.751165:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767983.751169:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767983.751179:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767983.751196:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742115328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767983.751197:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742115200:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767983.751223:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767983.751225:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767983.751226:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767983.751230:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3872 00000100:00080000:3.0:1775767983.751233:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767983.751235:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767983.751240:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767983.751250:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742115456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767983.751259:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767983.751264:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767983.751266:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767983.751270:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3872 00000100:00080000:3.0:1775767983.751284:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767983.751289:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775767983.751295:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775767983.751295:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767983.751313:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742115584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767983.751349:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767983/real 1775767983] req@ffff90c218da3480 x1862023742115200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767983.751368:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742115200/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767983.751377:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767983.751381:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767983.751384:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742115200:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767983.751408:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767983.751425:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767983.751434:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767983/real 1775767983] req@ffff90c218da1f80 x1862023742115328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767983.751443:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742115328/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767983.751449:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767983.751451:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767983.751453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767983.751458:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115456:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767983.751468:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767983.751474:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767983/real 1775767983] req@ffff90c218da0000 x1862023742115456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767983.751482:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742115456/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767983.751487:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767983.751489:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767983.751490:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115456:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767983.751494:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115584:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767983.751504:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767983.751509:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767983/real 1775767983] req@ffff90c218da3100 x1862023742115584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767983.751516:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742115584/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767983.751528:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767983.751531:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767983.751533:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115584:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767984.775074:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767985.799050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767986.823044:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767987.847013:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767988.870999:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767988.871008:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767988.871013:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767988.871021:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3877 00000100:00080000:3.0:1775767988.871032:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767988.871038:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767988.871067:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767988.871075:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742115712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767988.871096:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767988.871099:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767988.871100:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767988.871103:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3877 00000100:00080000:3.0:1775767988.871105:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767988.871108:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767988.871116:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767988.871126:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742115840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767988.871134:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775767988.871136:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742115712:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775767988.871137:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767988.871139:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767988.871148:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3877 00000100:00080000:3.0:1775767988.871150:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767988.871152:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767988.871155:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767988.871165:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742115968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767988.871171:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767988.871173:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767988.871175:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767988.871177:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3877 00000100:00080000:3.0:1775767988.871193:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767988.871196:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767988.871199:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775767988.871204:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775767988.871211:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742116096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767988.871243:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767988/real 1775767988] req@ffff90c218da0700 x1862023742115712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767988.871262:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742115712/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767988.871273:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767988.871276:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767988.871281:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742115712:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767988.871303:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115840:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767988.871321:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767988.871331:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767988/real 1775767988] req@ffff90c218da0a80 x1862023742115840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767988.871344:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742115840/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767988.871352:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767988.871355:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767988.871359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767988.871365:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115968:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767988.871379:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767988.871387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767988/real 1775767988] req@ffff90c218da0380 x1862023742115968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767988.871398:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742115968/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767988.871406:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767988.871409:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767988.871412:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742115968:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767988.871417:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116096:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767988.871431:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767988.871439:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767988/real 1775767988] req@ffff90c218da1880 x1862023742116096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767988.871451:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742116096/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767988.871460:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767988.871463:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767988.871466:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116096:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775767989.895088:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767990.919040:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767991.943037:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767992.967072:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775767993.990970:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767993.990981:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775767993.990986:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767993.990996:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3882 00000100:00080000:3.0:1775767993.991010:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767993.991019:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775767993.991037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775767993.991070:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742116224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767993.991106:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767993.991109:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775767993.991110:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767993.991113:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3882 00000100:00080000:3.0:1775767993.991116:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767993.991118:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767993.991127:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775767993.991138:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742116352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767993.991145:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767993.991149:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775767993.991150:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767993.991153:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3882 00000100:00080000:3.0:1775767993.991154:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767993.991159:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767993.991167:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775767993.991168:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742116224:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775767993.991177:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742116480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775767993.991188:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775767993.991191:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775767993.991192:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775767993.991196:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3882 00000100:00080000:3.0:1775767993.991200:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775767993.991203:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775767993.991208:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775767993.991219:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742116608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775767993.991243:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767993.991281:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffff90c218da3800 x1862023742116224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767993.991303:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742116224/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767993.991315:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767993.991320:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767993.991324:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742116224:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767993.991333:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767993.991357:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767993.991370:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffff90c218da2d80 x1862023742116352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767993.991383:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742116352/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767993.991392:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767993.991395:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767993.991399:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767993.991425:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116480:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767993.991447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767993.991455:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffff90c218da0e00 x1862023742116480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767993.991463:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742116480/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767993.991473:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767993.991476:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767993.991478:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116480:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767993.991482:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116608:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767993.991493:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767993.991499:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767993/real 1775767993] req@ffff90c218da1880 x1862023742116608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767993.991506:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742116608/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768018 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767993.991512:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767993.991514:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767993.991516:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116608:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775767995.015018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767996.039088:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775767997.063127:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775767998.087149:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767999.111010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767999.111021:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775767999.111026:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767999.111038:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3887 00010000:00010000:3.0:1775767999.111048:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775767999.111051:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767999.111062:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775767999.111114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742116736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767999.111146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767999.111151:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775767999.111154:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767999.111159:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3887 00000100:00080000:1.0:1775767999.111165:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767999.111170:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775767999.111177:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742116736:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775767999.111183:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767999.111201:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742116864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767999.111211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767999.111214:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775767999.111215:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775767999.111219:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3887 00000100:00080000:1.0:1775767999.111223:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767999.111225:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767999.111229:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775767999.111236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742116992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775767999.111243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775767999.111245:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775767999.111247:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775767999.111250:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775767999.111252:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3887 00000100:00080000:1.0:1775767999.111256:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775767999.111258:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775767999.111262:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775767999.111272:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742117120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775767999.111295:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767999/real 1775767999] req@ffff90c203494e00 x1862023742116736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767999.111310:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742116736/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767999.111318:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767999.111322:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767999.111326:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742116736:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775767999.111348:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116864:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767999.111363:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767999.111372:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767999/real 1775767999] req@ffff90c203496d80 x1862023742116864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767999.111380:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742116864/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767999.111385:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767999.111388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767999.111390:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767999.111393:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116992:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775767999.111403:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767999.111408:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767999/real 1775767999] req@ffff90c203497100 x1862023742116992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767999.111415:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742116992/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767999.111420:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767999.111422:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767999.111424:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742116992:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775767999.111427:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117120:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775767999.111437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775767999.111442:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775767999/real 1775767999] req@ffff90c203495880 x1862023742117120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775767999.111450:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742117120/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775767999.111458:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775767999.111459:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775767999.111461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117120:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768000.135126:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768001.159048:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768002.183068:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768003.207055:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768004.230991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768004.231001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768004.231006:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768004.231016:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3892 00000100:00080000:1.0:1775768004.231028:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768004.231035:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775768004.231045:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775768004.231073:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742117248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768004.231099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768004.231103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768004.231105:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768004.231110:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3892 00000100:00080000:1.0:1775768004.231114:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768004.231119:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768004.231128:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742117248:192.168.202.155@tcp:250:kworker.0 00000008:00100000:1.0:1775768004.231129:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768004.231145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742117376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768004.231157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768004.231161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768004.231163:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768004.231167:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3892 00000100:00080000:1.0:1775768004.231171:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768004.231175:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768004.231180:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775768004.231186:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775768004.231192:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742117504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768004.231203:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768004.231207:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768004.231209:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768004.231214:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3892 00000100:00080000:1.0:1775768004.231218:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775768004.231219:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768004/real 1775768004] req@ffff90c203495c00 x1862023742117248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768004.231226:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768004.231235:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768004.231241:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742117248/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768004.231249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742117632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768004.231252:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768004.231261:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768004.231267:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742117248:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768004.231288:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768004.231301:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768004.231308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768004/real 1775768004] req@ffff90c203497b80 x1862023742117376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768004.231316:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742117376/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768004.231326:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768004.231328:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768004.231331:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768004.231334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117504:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768004.231343:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768004.231348:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768004/real 1775768004] req@ffff90c203496300 x1862023742117504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768004.231355:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742117504/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768004.231360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768004.231362:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768004.231363:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117504:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768004.231366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117632:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768004.231376:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768004.231381:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768004/real 1775768004] req@ffff90c203496a00 x1862023742117632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768004.231387:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742117632/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768004.231393:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768004.231395:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768004.231398:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117632:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768005.255013:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768006.279043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768007.303050:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768008.327021:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768009.350937:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768009.350943:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768009.350947:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768009.350954:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3897 00000100:00080000:1.0:1775768009.350963:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768009.350969:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775768009.350995:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775768009.351000:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742117760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768009.351020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768009.351023:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768009.351024:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768009.351026:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3897 00000100:00080000:1.0:1775768009.351029:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768009.351031:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768009.351037:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768009.351045:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742117888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768009.351049:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742117760:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775768009.351057:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768009.351060:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768009.351060:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768009.351063:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3897 00000100:00080000:1.0:1775768009.351065:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768009.351067:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768009.351070:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768009.351079:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742118016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768009.351087:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768009.351090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768009.351091:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768009.351095:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3897 00000100:00080000:1.0:1775768009.351099:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768009.351102:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775768009.351107:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775768009.351110:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768009.351123:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742118144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768009.351147:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768009/real 1775768009] req@ffff90c203496680 x1862023742117760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768009.351162:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742117760/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768009.351169:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768009.351172:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768009.351174:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742117760:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768009.351190:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768009.351204:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768009.351211:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768009/real 1775768009] req@ffff90c203494a80 x1862023742117888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768009.351226:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742117888/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768009.351232:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768009.351234:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768009.351236:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742117888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768009.351239:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118016:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768009.351249:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768009.351256:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768009/real 1775768009] req@ffff90c203494700 x1862023742118016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768009.351262:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742118016/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768009.351267:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768009.351268:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768009.351270:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118016:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768009.351273:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118144:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768009.351282:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768009.351286:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768009/real 1775768009] req@ffff90c203495500 x1862023742118144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768009.351292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742118144/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768009.351298:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768009.351299:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768009.351301:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118144:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768010.375105:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768011.399013:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768012.423027:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768013.446972:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768014.470954:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768014.470961:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768014.470965:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768014.470973:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3902 00000100:00080000:3.0:1775768014.470983:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775768014.470986:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768014.471008:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768014.471043:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742118272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768014.471063:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768014.471066:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768014.471067:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768014.471070:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3902 00000100:00080000:3.0:1775768014.471073:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768014.471076:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768014.471083:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768014.471092:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742118400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768014.471095:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742118272:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768014.471103:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768014.471105:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768014.471106:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768014.471109:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3902 00000100:00080000:3.0:1775768014.471111:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768014.471113:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768014.471116:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768014.471121:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742118528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768014.471127:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768014.471129:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768014.471130:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768014.471132:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3902 00000100:00080000:3.0:1775768014.471134:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768014.471136:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768014.471140:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768014.471147:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742118656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768014.471160:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768014.471211:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768014/real 1775768014] req@ffff90c218da0380 x1862023742118272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768014.471227:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742118272/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768014.471234:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768014.471236:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768014.471239:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742118272:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768014.471257:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118400:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768014.471272:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768014.471279:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768014/real 1775768014] req@ffff90c218da0a80 x1862023742118400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768014.471287:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742118400/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768014.471292:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768014.471294:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768014.471295:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768014.471298:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118528:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768014.471307:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768014.471311:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768014/real 1775768014] req@ffff90c218da0700 x1862023742118528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768014.471317:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742118528/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768014.471321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768014.471323:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768014.471324:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118528:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768014.471328:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118656:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768014.471337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768014.471341:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768014/real 1775768014] req@ffff90c218da3100 x1862023742118656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768014.471347:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742118656/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768014.471353:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768014.471354:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768014.471356:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118656:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768015.495007:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768016.518990:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768017.543053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768018.567012:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768019.590965:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768019.590974:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768019.590979:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768019.590988:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3908 00000100:00080000:3.0:1775768019.590997:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768019.591003:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768019.591030:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768019.591034:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742118784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768019.591054:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768019.591056:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768019.591057:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768019.591059:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3908 00000100:00080000:3.0:1775768019.591062:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768019.591064:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768019.591071:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768019.591080:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742118784:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768019.591081:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742118912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768019.591093:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768019.591095:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768019.591096:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768019.591099:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3908 00000100:00080000:3.0:1775768019.591101:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768019.591103:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768019.591106:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768019.591113:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742119040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768019.591118:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768019.591119:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768019.591120:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768019.591122:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3908 00000100:00080000:3.0:1775768019.591130:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768019.591133:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775768019.591137:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775768019.591138:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768019.591147:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742119168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768019.591171:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768019/real 1775768019] req@ffff90c218da0000 x1862023742118784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768019.591183:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742118784/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768019.591190:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768019.591193:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768019.591195:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742118784:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768019.591212:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768019.591225:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768019.591232:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768019/real 1775768019] req@ffff90c218da1f80 x1862023742118912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768019.591240:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742118912/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768019.591245:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768019.591247:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768019.591249:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742118912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768019.591252:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119040:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768019.591261:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768019.591265:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768019/real 1775768019] req@ffff90c218da3480 x1862023742119040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768019.591272:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742119040/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768019.591276:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768019.591277:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768019.591279:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119040:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768019.591282:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119168:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768019.591290:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768019.591295:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768019/real 1775768019] req@ffff90c218da3100 x1862023742119168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768019.591301:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742119168/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768019.591311:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768019.591312:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768019.591314:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119168:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768020.615053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768021.639042:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768022.663048:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768023.687056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768024.710991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768024.711001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768024.711006:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768024.711016:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3913 00000100:00080000:3.0:1775768024.711026:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768024.711032:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768024.711068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742119296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775768024.711079:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768024.711091:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768024.711094:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768024.711096:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768024.711099:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3913 00000100:00080000:3.0:1775768024.711101:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768024.711103:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768024.711123:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768024.711129:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742119296:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768024.711135:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742119424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768024.711144:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768024.711146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768024.711148:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768024.711151:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3913 00000100:00080000:3.0:1775768024.711153:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768024.711155:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768024.711159:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768024.711167:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742119552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768024.711173:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768024.711175:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768024.711177:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768024.711180:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3913 00000100:00080000:3.0:1775768024.711183:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768024.711186:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768024.711189:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768024.711197:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742119680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768024.711227:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768024.711250:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768024.711265:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119680:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768025.735029:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768026.759055:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768027.783117:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775768027.783214:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775768027.783267:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775768027.783286:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775768027.783333:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775768027.783343:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775768027.783351:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775768027.783419:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768024/real 1775768027] req@ffff90c218da0a80 x1862023742119424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768027.783452:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742119424/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768027.783466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768027.783471:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768027.783475:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768027.783485:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768024/real 1775768027] req@ffff90c218da0380 x1862023742119552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768027.783499:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742119552/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768027.783506:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768027.783511:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768027.783514:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119552:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768027.783520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768024/real 1775768027] req@ffff90c218da1880 x1862023742119680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768027.783532:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742119680/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768027.783544:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768027.783548:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768027.783552:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119680:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775768027.783560:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768024/real 1775768027] req@ffff90c218da0700 x1862023742119296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768027.783573:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742119296/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768027.783580:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768027.783583:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768027.783586:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742119296:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775768028.807018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768029.830945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768029.830954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768029.830958:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768029.830968:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3918 00000100:00080000:3.0:1775768029.830980:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768029.830988:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768029.830998:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768029.831026:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742119808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768029.831050:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768029.831053:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768029.831055:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768029.831059:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3918 00000100:00080000:3.0:1775768029.831063:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768029.831068:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768029.831078:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768029.831080:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742119808:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768029.831094:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742119936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768029.831105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768029.831108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768029.831110:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768029.831115:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3918 00000100:00080000:3.0:1775768029.831118:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768029.831123:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768029.831129:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768029.831145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742120064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768029.831152:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775768029.831160:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768029.831163:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768029.831165:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768029.831171:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3918 00000100:00080000:3.0:1775768029.831178:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768029.831183:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768029.831188:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768029.831204:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768029/real 1775768029] req@ffff90c218da0e00 x1862023742119808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768029.831204:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742120192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768029.831235:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742119808/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768029.831259:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768029.831264:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768029.831269:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742119808:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768029.831300:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119936:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768029.831326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768029.831340:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768029/real 1775768029] req@ffff90c218da2d80 x1862023742119936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768029.831353:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742119936/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768029.831369:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768029.831374:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768029.831377:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742119936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768029.831384:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120064:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768029.831413:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768029.831425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768029/real 1775768029] req@ffff90c218da3800 x1862023742120064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768029.831436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742120064/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768029.831445:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768029.831448:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768029.831452:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120064:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768029.831459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120192:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768029.831483:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768029.831495:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768029/real 1775768029] req@ffff90c218da0700 x1862023742120192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768029.831509:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742120192/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768029.831520:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768029.831524:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768029.831527:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120192:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768030.855051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768031.879026:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768032.903033:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768033.927002:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768034.950956:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768034.950971:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768034.950975:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768034.950983:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3923 00000100:00080000:3.0:1775768034.950993:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768034.950999:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768034.951020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768034.951033:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742120320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768034.951052:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768034.951055:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768034.951057:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768034.951059:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3923 00000100:00080000:3.0:1775768034.951061:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768034.951064:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768034.951072:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768034.951081:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742120448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768034.951086:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742120320:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768034.951100:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768034.951102:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768034.951103:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768034.951106:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3923 00000100:00080000:3.0:1775768034.951108:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768034.951111:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768034.951115:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768034.951124:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742120576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768034.951131:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768034.951133:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768034.951134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768034.951137:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3923 00000100:00080000:3.0:1775768034.951139:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768034.951141:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768034.951145:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768034.951148:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768034.951157:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742120704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768034.951185:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768034/real 1775768034] req@ffff90c218da1880 x1862023742120320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768034.951207:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742120320/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768034.951215:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768034.951218:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768034.951221:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742120320:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768034.951239:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768034.951254:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768034.951262:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768034/real 1775768034] req@ffff90c218da0380 x1862023742120448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768034.951272:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742120448/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768034.951277:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768034.951279:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768034.951281:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768034.951285:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120576:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768034.951294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768034.951300:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768034/real 1775768034] req@ffff90c218da0a80 x1862023742120576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768034.951314:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742120576/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768034.951319:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768034.951321:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768034.951323:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120576:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768034.951326:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120704:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768034.951337:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768034.951342:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768034/real 1775768034] req@ffff90c218da3100 x1862023742120704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768034.951349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742120704/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768059 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768034.951360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768034.951362:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768034.951364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120704:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768035.975012:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768036.999008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768038.023035:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768039.047011:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768040.070975:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768040.070986:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768040.070991:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768040.071001:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3928 00000100:00080000:1.0:1775768040.071013:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775768040.071016:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768040.071021:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768040.071061:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742120832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768040.071095:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768040.071099:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768040.071101:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768040.071106:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3928 00000100:00080000:1.0:1775768040.071111:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768040.071116:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768040.071125:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768040.071135:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742120832:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775768040.071143:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742120960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768040.071156:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768040.071159:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768040.071161:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768040.071165:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3928 00000100:00080000:1.0:1775768040.071169:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768040.071174:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768040.071179:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768040.071192:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742121088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768040.071204:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1775768040.071204:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775768040.071209:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768040.071211:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768040.071217:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3928 00000100:00080000:1.0:1775768040.071222:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768040.071227:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768040.071234:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768040.071240:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768040/real 1775768040] req@ffff90c203495180 x1862023742120832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768040.071249:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742121216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768040.071258:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742120832/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768040.071269:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768040.071273:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768040.071276:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742120832:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768040.071295:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120960:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768040.071310:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768040.071317:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768040/real 1775768040] req@ffff90c203495500 x1862023742120960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768040.071325:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742120960/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768040.071330:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768040.071332:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768040.071334:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742120960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768040.071338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121088:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768040.071348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768040.071353:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768040/real 1775768040] req@ffff90c203494700 x1862023742121088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768040.071361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742121088/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768040.071373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768040.071376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768040.071378:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121088:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768040.071382:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121216:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768040.071392:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768040.071398:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768040/real 1775768040] req@ffff90c203494a80 x1862023742121216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768040.071405:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742121216/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768040.071412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768040.071414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768040.071415:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121216:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768041.095047:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768042.119031:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768043.143065:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768044.167038:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768045.190995:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768045.191007:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768045.191013:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768045.191028:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3933 00000100:00080000:3.0:1775768045.191043:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775768045.191047:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768045.191053:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768045.191102:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742121344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768045.191134:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768045.191138:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768045.191140:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768045.191144:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3933 00000100:00080000:3.0:1775768045.191148:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768045.191153:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768045.191165:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768045.191170:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742121344:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768045.191181:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742121472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768045.191194:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768045.191198:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768045.191200:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768045.191204:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3933 00000100:00080000:3.0:1775768045.191232:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768045.191236:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768045.191242:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775768045.191244:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768045.191257:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742121600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768045.191268:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768045.191272:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768045.191275:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768045.191280:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3933 00000100:00080000:3.0:1775768045.191285:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775768045.191288:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768045/real 1775768045] req@ffff90c218da3480 x1862023742121344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768045.191292:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768045.191301:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768045.191312:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742121344/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768045.191321:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1775768045.191322:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742121728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768045.191328:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768045.191335:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742121344:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768045.191363:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768045.191378:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768045.191385:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768045/real 1775768045] req@ffff90c218da1f80 x1862023742121472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768045.191393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742121472/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768045.191399:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768045.191401:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768045.191403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768045.191406:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121600:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768045.191416:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768045.191422:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768045/real 1775768045] req@ffff90c218da0000 x1862023742121600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768045.191448:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742121600/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768045.191458:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768045.191461:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768045.191463:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121600:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768045.191466:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121728:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768045.191481:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768045.191487:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768045/real 1775768045] req@ffff90c218da3100 x1862023742121728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768045.191494:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742121728/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768045.191501:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768045.191503:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768045.191505:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121728:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768046.215048:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768047.239051:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768048.263081:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768049.287080:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768050.310917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768050.310926:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768050.310932:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768050.310941:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3938 00000100:00080000:3.0:1775768050.310953:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768050.310960:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768050.311004:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742121856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775768050.311018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768050.311035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768050.311039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768050.311042:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768050.311046:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3938 00000100:00080000:3.0:1775768050.311051:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768050.311055:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768050.311066:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768050.311074:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742121856:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768050.311079:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742121984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768050.311093:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768050.311095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768050.311096:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768050.311100:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3938 00000100:00080000:3.0:1775768050.311102:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768050.311105:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768050.311108:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768050.311114:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742122112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768050.311120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768050.311123:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768050.311124:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768050.311127:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3938 00000100:00080000:3.0:1775768050.311145:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768050.311147:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768050.311150:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768050.311159:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768050.311161:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742122240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768050.311207:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768050/real 1775768050] req@ffff90c218da0a80 x1862023742121856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768050.311227:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742121856/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768050.311237:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768050.311242:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768050.311246:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742121856:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768050.311280:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768050.311313:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768050.311325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768050/real 1775768050] req@ffff90c218da0380 x1862023742121984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768050.311337:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742121984/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768050.311351:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768050.311355:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768050.311359:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742121984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768050.311364:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122112:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768050.311413:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768050.311424:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768050/real 1775768050] req@ffff90c218da1880 x1862023742122112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768050.311435:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742122112/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768050.311443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768050.311446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768050.311449:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122112:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768050.311454:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122240:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768050.311473:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768050.311481:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768050/real 1775768050] req@ffff90c218da0700 x1862023742122240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768050.311492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742122240/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768050.311502:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768050.311505:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768050.311508:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122240:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768051.335082:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768052.358992:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768053.383046:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768054.407022:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768055.430958:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768055.430966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768055.430971:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768055.430980:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3943 00000100:00080000:3.0:1775768055.430992:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768055.431006:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768055.431017:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768055.431043:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742122368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768055.431064:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768055.431067:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768055.431068:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768055.431071:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3943 00000100:00080000:3.0:1775768055.431073:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768055.431076:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768055.431084:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768055.431093:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742122496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768055.431101:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742122368:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768055.431101:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768055.431105:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768055.431108:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768055.431112:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3943 00000100:00080000:3.0:1775768055.431114:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768055.431115:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768055.431118:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768055.431126:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742122624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768055.431131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768055.431134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768055.431135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768055.431137:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3943 00000100:00080000:3.0:1775768055.431165:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768055.431167:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768055.431170:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768055.431177:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768055.431179:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742122752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768055.431218:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768055/real 1775768055] req@ffff90c218da3800 x1862023742122368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768055.431236:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742122368/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768055.431246:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768055.431250:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768055.431254:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742122368:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768055.431274:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122496:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768055.431290:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768055.431298:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768055/real 1775768055] req@ffff90c218da2d80 x1862023742122496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768055.431305:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742122496/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768055.431314:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768055.431316:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768055.431318:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768055.431321:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122624:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768055.431359:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768055.431366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768055/real 1775768055] req@ffff90c218da0e00 x1862023742122624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768055.431373:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742122624/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768055.431378:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768055.431380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768055.431382:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122624:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768055.431386:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122752:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768055.431397:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768055.431401:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768055/real 1775768055] req@ffff90c218da0700 x1862023742122752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768055.431408:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742122752/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768055.431413:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768055.431414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768055.431417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742122752:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768056.455057:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768057.479018:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768058.503000:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768059.527086:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768060.550995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768060.551005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768060.551009:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768060.551018:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3949 00010000:00010000:3.0:1775768060.551021:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768060.551028:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768060.551033:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768060.551065:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742122880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768060.551089:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768060.551092:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768060.551094:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768060.551098:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3949 00000100:00080000:1.0:1775768060.551103:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768060.551107:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768060.551115:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768060.551126:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742122880:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775768060.551127:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742123008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768060.551141:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768060.551143:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768060.551144:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768060.551154:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3949 00000100:00080000:1.0:1775768060.551158:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768060.551162:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768060.551166:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768060.551177:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742123136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768060.551187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768060.551191:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768060.551195:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775768060.551195:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775768060.551202:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3949 00000100:00080000:1.0:1775768060.551205:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768060.551207:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768060.551211:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768060.551219:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742123264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768060.551239:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768060/real 1775768060] req@ffff90c203496680 x1862023742122880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768060.551258:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742122880/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768060.551268:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768060.551273:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768060.551277:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742122880:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768060.551330:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768060.551353:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768060.551366:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768060/real 1775768060] req@ffff90c203496a00 x1862023742123008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768060.551378:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742123008/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768060.551386:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768060.551389:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768060.551392:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768060.551398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123136:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768060.551415:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768060.551425:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768060/real 1775768060] req@ffff90c203496300 x1862023742123136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768060.551436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742123136/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768060.551443:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768060.551446:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768060.551449:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123136:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768060.551455:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123264:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768060.551474:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768060.551485:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768060/real 1775768060] req@ffff90c203497b80 x1862023742123264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768060.551496:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742123264/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768060.551505:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768060.551508:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768060.551511:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123264:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768061.575064:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768062.599058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768063.623061:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768064.647077:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768065.671014:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:1.0:1775768065.671017:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768065.671025:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768065.671030:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768065.671044:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3954 00000100:00080000:3.0:1775768065.671054:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768065.671060:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768065.671088:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742123392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768065.671109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768065.671111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768065.671113:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768065.671115:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3954 00000100:00080000:3.0:1775768065.671117:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768065.671119:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768065.671127:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768065.671134:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742123520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768065.671142:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768065.671144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768065.671145:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768065.671148:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3954 00000100:00080000:3.0:1775768065.671150:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775768065.671155:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742123392:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768065.671157:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768065.671167:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768065.671176:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742123648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768065.671182:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768065.671184:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768065.671184:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768065.671189:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3954 00000100:00080000:3.0:1775768065.671192:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768065.671194:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768065.671197:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768065.671206:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742123776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768065.671257:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768065.671296:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768065/real 1775768065] req@ffff90c218da1880 x1862023742123392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768065.671310:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742123392/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768065.671319:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768065.671322:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768065.671325:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742123392:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768065.671347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123520:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768065.671361:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768065.671367:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768065/real 1775768065] req@ffff90c218da0380 x1862023742123520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768065.671375:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742123520/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768065.671380:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768065.671389:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768065.671391:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768065.671395:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123648:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768065.671408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768065.671413:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768065/real 1775768065] req@ffff90c218da0a80 x1862023742123648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768065.671420:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742123648/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768065.671425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768065.671427:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768065.671428:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123648:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768065.671432:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123776:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768065.671442:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768065.671446:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768065/real 1775768065] req@ffff90c218da3100 x1862023742123776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768065.671453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742123776/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768065.671459:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768065.671461:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768065.671462:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742123776:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768066.695028:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768067.718991:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768068.743061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768069.767028:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768070.791010:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768070.791022:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768070.791027:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768070.791038:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3959 00000100:00080000:3.0:1775768070.791051:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768070.791059:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768070.791094:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768070.791108:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742123904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768070.791133:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768070.791136:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768070.791137:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768070.791140:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3959 00000100:00080000:3.0:1775768070.791143:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768070.791151:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768070.791161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768070.791175:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742124032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768070.791176:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742123904:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768070.791201:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768070.791204:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768070.791205:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768070.791209:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3959 00000100:00080000:3.0:1775768070.791212:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768070.791215:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768070.791220:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768070.791232:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742124160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768070.791243:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768070.791246:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768070.791248:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768070.791252:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3959 00000100:00080000:3.0:1775768070.791258:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768070.791263:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775768070.791270:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775768070.791281:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768070.791299:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742124288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768070.791325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768070/real 1775768070] req@ffff90c218da0000 x1862023742123904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768070.791349:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742123904/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768070.791361:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768070.791366:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768070.791371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742123904:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768070.791411:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768070.791437:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768070.791451:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768070/real 1775768070] req@ffff90c218da1f80 x1862023742124032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768070.791465:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742124032/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768070.791475:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768070.791479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768070.791483:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768070.791489:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124160:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768070.791510:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768070.791520:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768070/real 1775768070] req@ffff90c218da3480 x1862023742124160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768070.791534:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742124160/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768070.791543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768070.791546:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768070.791550:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124160:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768070.791557:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124288:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768070.791576:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768070.791586:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768070/real 1775768070] req@ffff90c218da3100 x1862023742124288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768070.791600:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742124288/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768070.791619:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768070.791623:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768070.791627:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124288:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768071.815039:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768072.839086:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768073.863147:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768074.887111:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768075.911024:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768075.911033:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768075.911039:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768075.911050:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3964 00000100:00080000:3.0:1775768075.911060:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768075.911066:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768075.911102:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742124416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775768075.911116:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768075.911146:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768075.911150:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768075.911153:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768075.911158:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3964 00000100:00080000:3.0:1775768075.911164:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768075.911170:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768075.911181:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768075.911197:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742124416:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768075.911199:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742124544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768075.911221:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768075.911223:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768075.911225:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768075.911229:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3964 00000100:00080000:3.0:1775768075.911234:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768075.911237:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768075.911242:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768075.911251:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742124672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768075.911258:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768075.911261:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768075.911263:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768075.911265:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3964 00000100:00080000:3.0:1775768075.911279:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768075.911281:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768075.911287:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768075.911292:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768075.911303:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742124800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768075.911346:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768075/real 1775768075] req@ffff90c218da0a80 x1862023742124416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768075.911370:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742124416/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768075.911382:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768075.911388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768075.911393:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742124416:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768075.911423:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124544:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768075.911444:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768075.911454:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768075/real 1775768075] req@ffff90c218da0380 x1862023742124544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768075.911463:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742124544/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768075.911469:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768075.911471:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768075.911473:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768075.911478:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124672:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768075.911489:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768075.911495:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768075/real 1775768075] req@ffff90c218da1880 x1862023742124672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768075.911502:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742124672/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768075.911507:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768075.911509:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768075.911511:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124672:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768075.911514:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124800:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768075.911524:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768075.911530:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768075/real 1775768075] req@ffff90c218da0700 x1862023742124800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768075.911536:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742124800/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768075.911543:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768075.911545:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768075.911546:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742124800:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768076.935076:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768077.959056:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768078.983069:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768080.007064:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768081.030949:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768081.030960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768081.030965:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768081.030975:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3969 00000100:00080000:3.0:1775768081.030987:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768081.030994:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768081.031032:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742124928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775768081.031044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768081.031054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768081.031057:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768081.031059:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768081.031062:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3969 00000100:00080000:3.0:1775768081.031065:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768081.031067:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768081.031075:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768081.031096:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742125056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768081.031105:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768081.031108:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768081.031109:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775768081.031114:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742124928:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768081.031115:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3969 00000100:00080000:3.0:1775768081.031121:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768081.031124:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768081.031129:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768081.031138:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742125184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768081.031145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768081.031147:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768081.031149:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768081.031152:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3969 00000100:00080000:3.0:1775768081.031165:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768081.031167:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768081.031171:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768081.031175:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768081.031183:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742125312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768081.031209:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768081/real 1775768081] req@ffff90c218da0e00 x1862023742124928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768081.031224:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742124928/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768081.031232:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768081.031235:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768081.031238:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742124928:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768081.031256:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125056:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768081.031269:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768081.031276:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768081/real 1775768081] req@ffff90c218da2d80 x1862023742125056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768081.031284:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742125056/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768081.031289:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768081.031291:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768081.031294:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768081.031297:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125184:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768081.031316:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768081.031322:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768081/real 1775768081] req@ffff90c218da3800 x1862023742125184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768081.031329:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742125184/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768081.031338:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768081.031340:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768081.031342:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125184:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768081.031346:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125312:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768081.031355:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768081.031360:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768081/real 1775768081] req@ffff90c218da0700 x1862023742125312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768081.031366:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742125312/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768081.031373:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768081.031374:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768081.031376:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125312:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768082.055040:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768083.079081:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768084.103115:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768085.126989:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768086.150977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768086.150985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768086.150989:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768086.150998:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3974 00000100:00080000:3.0:1775768086.151008:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768086.151014:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768086.151062:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742125440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775768086.151070:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768086.151085:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768086.151088:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768086.151090:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768086.151093:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3974 00000100:00080000:3.0:1775768086.151097:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768086.151101:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768086.151109:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768086.151118:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742125568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768086.151128:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742125440:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768086.151129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768086.151134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768086.151137:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768086.151141:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3974 00000100:00080000:3.0:1775768086.151143:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768086.151145:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768086.151148:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768086.151157:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742125696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768086.151163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768086.151166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768086.151167:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768086.151170:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3974 00000100:00080000:3.0:1775768086.151181:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768086.151183:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768086.151187:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768086.151190:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768086.151200:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742125824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768086.151227:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768086/real 1775768086] req@ffff90c218da1880 x1862023742125440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768086.151242:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742125440/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768086.151252:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768086.151256:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768086.151260:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742125440:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768086.151296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768086.151316:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768086.151325:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768086/real 1775768086] req@ffff90c218da0380 x1862023742125568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768086.151333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742125568/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768086.151339:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768086.151341:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768086.151343:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768086.151347:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125696:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768086.151371:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768086.151381:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768086/real 1775768086] req@ffff90c218da0a80 x1862023742125696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768086.151394:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742125696/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768086.151403:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768086.151406:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768086.151409:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125696:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768086.151416:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125824:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768086.151433:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768086.151440:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768086/real 1775768086] req@ffff90c218da3100 x1862023742125824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768086.151449:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742125824/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768086.151456:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768086.151458:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768086.151460:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742125824:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768087.175060:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768088.199043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768089.222990:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768090.247071:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768091.271039:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768091.271053:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768091.271059:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768091.271072:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3979 00000100:00080000:0.0:1775768091.271088:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768091.271097:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768091.271113:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768091.271146:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742125952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768091.271175:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768091.271178:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768091.271180:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768091.271185:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3979 00000100:00080000:0.0:1775768091.271188:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768091.271192:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768091.271199:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742125952:192.168.202.155@tcp:250:kworker.0 00000008:00100000:0.0:1775768091.271203:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768091.271223:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742126080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768091.271231:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768091.271235:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768091.271236:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768091.271239:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3979 00000100:00080000:0.0:1775768091.271242:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768091.271245:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768091.271248:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768091.271256:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742126208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768091.271263:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768091.271266:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768091.271268:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768091.271272:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3979 00000100:00080000:0.0:1775768091.271275:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768091.271278:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768091.271282:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768091.271291:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742126336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768091.271314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768091.271338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768091.271357:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126336:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768092.295109:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768093.319101:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768094.343257:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775768094.343386:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775768094.343442:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775768094.343468:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775768094.343526:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775768094.343536:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775768094.343543:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775768094.343617:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768091/real 1775768094] req@ffff90c203638000 x1862023742126080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768094.343644:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742126080/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768094.343657:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768094.343663:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768094.343667:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768094.343675:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768091/real 1775768094] req@ffff90c203639180 x1862023742126208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768094.343686:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742126208/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768094.343696:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768094.343699:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768094.343702:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126208:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768094.343708:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768091/real 1775768094] req@ffff90c20363aa00 x1862023742126336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768094.343719:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742126336/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768094.343728:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768094.343731:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768094.343734:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126336:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775768094.343739:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768091/real 1775768094] req@ffff90c203638e00 x1862023742125952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768094.343750:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742125952/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768094.343756:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768094.343759:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768094.343762:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742125952:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775768095.367072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768096.391035:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768096.391045:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768096.391050:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768096.391061:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3984 00010000:00010000:2.0:1775768096.391073:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768096.391075:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768096.391084:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768096.391126:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742126464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768096.391151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768096.391154:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768096.391155:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768096.391157:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3984 00000100:00080000:3.0:1775768096.391161:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768096.391165:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768096.391177:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768096.391188:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742126464:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768096.391194:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742126592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768096.391208:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768096.391212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768096.391215:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768096.391220:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3984 00000100:00080000:3.0:1775768096.391224:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768096.391228:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768096.391234:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768096.391249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742126720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768096.391260:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768096.391265:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768096.391267:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768096.391272:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3984 00000100:00080000:3.0:1775768096.391276:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775768096.391282:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775768096.391285:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768096.391292:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768096.391309:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742126848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768096.391339:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768096/real 1775768096] req@ffff90c218da3480 x1862023742126464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768096.391364:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742126464/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768096.391376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768096.391381:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768096.391385:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742126464:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768096.391418:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126592:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768096.391443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768096.391456:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768096/real 1775768096] req@ffff90c218da1f80 x1862023742126592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768096.391469:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742126592/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768096.391478:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768096.391481:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768096.391484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768096.391491:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126720:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768096.391510:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768096.391522:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768096/real 1775768096] req@ffff90c218da0000 x1862023742126720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768096.391534:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742126720/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768096.391547:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768096.391550:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768096.391554:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126720:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768096.391559:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126848:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768096.391586:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768096.391597:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768096/real 1775768096] req@ffff90c218da3100 x1862023742126848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768096.391608:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742126848/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768096.391619:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768096.391622:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768096.391625:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742126848:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768097.415034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768098.439072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768099.463031:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768100.487000:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768101.510961:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768101.510970:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768101.510975:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768101.510985:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3989 00000100:00080000:3.0:1775768101.510997:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768101.511006:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768101.511025:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768101.511044:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742126976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768101.511069:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768101.511072:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768101.511074:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768101.511077:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3989 00000100:00080000:3.0:1775768101.511081:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768101.511085:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768101.511094:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768101.511107:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742127104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768101.511111:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742126976:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768101.511123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768101.511126:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768101.511128:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768101.511133:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3989 00000100:00080000:3.0:1775768101.511136:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768101.511140:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768101.511146:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768101.511156:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742127232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768101.511166:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768101.511169:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768101.511170:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775768101.511175:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775768101.511177:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3989 00000100:00080000:3.0:1775768101.511182:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768101.511186:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768101.511191:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768101.511202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742127360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768101.511218:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768101/real 1775768101] req@ffff90c218da0a80 x1862023742126976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768101.511238:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742126976/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768101.511248:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768101.511253:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768101.511257:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742126976:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768101.511279:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768101.511300:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768101.511312:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768101/real 1775768101] req@ffff90c218da0380 x1862023742127104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768101.511325:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742127104/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768101.511333:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768101.511336:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768101.511339:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768101.511346:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127232:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768101.511365:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768101.511374:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768101/real 1775768101] req@ffff90c218da1880 x1862023742127232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768101.511385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742127232/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768101.511393:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768101.511396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768101.511399:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127232:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768101.511405:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127360:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768101.511424:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768101.511434:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768101/real 1775768101] req@ffff90c218da0700 x1862023742127360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768101.511445:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742127360/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768101.511455:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768101.511458:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768101.511461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127360:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775768102.534980:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768103.558991:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768104.582998:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768105.606968:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768106.630969:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768106.630980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768106.630985:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768106.630997:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 3995 00010000:00010000:3.0:1775768106.630997:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768106.631010:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768106.631017:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775768106.631052:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742127488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768106.631079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768106.631084:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768106.631086:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768106.631090:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3995 00000100:00080000:0.0:1775768106.631095:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768106.631100:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768106.631108:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768106.631124:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742127616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768106.631132:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742127488:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775768106.631139:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768106.631144:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768106.631146:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768106.631151:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3995 00000100:00080000:0.0:1775768106.631155:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768106.631159:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768106.631165:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768106.631178:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742127744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768106.631189:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768106.631195:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1775768106.631195:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775768106.631199:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768106.631205:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 3995 00000100:00080000:0.0:1775768106.631211:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768106.631215:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768106.631221:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768106.631231:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768106/real 1775768106] req@ffff90c203638700 x1862023742127488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768106.631237:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742127872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768106.631255:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742127488/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768106.631272:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768106.631275:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768106.631278:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742127488:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768106.631296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127616:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768106.631317:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768106.631326:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768106/real 1775768106] req@ffff90c203638e00 x1862023742127616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768106.631340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742127616/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768106.631345:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768106.631348:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768106.631350:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768106.631353:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127744:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768106.631365:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768106.631370:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768106/real 1775768106] req@ffff90c20363aa00 x1862023742127744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768106.631377:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742127744/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768106.631382:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768106.631384:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768106.631386:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127744:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768106.631389:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127872:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768106.631408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768106.631414:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768106/real 1775768106] req@ffff90c203639180 x1862023742127872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768106.631421:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742127872/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768106.631432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768106.631434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768106.631436:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742127872:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768107.655051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768108.679038:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768109.703095:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768110.726988:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768111.750972:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768111.750990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768111.750994:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:2.0:1775768111.751002:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768111.751003:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4000 00000100:00080000:0.0:1775768111.751014:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768111.751019:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775768111.751051:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742128000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768111.751073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768111.751075:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768111.751077:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768111.751079:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4000 00000100:00080000:0.0:1775768111.751085:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768111.751089:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768111.751099:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768111.751109:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742128128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768111.751109:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742128000:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775768111.751123:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768111.751126:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768111.751127:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768111.751130:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4000 00000100:00080000:0.0:1775768111.751132:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768111.751134:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768111.751138:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768111.751146:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742128256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768111.751158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768111.751161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768111.751162:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768111.751165:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4000 00000100:00080000:0.0:1775768111.751168:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775768111.751170:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775768111.751174:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768111.751178:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768111.751186:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742128384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768111.751206:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768111/real 1775768111] req@ffff90c203638000 x1862023742128000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768111.751222:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742128000/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768111.751231:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768111.751235:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768111.751241:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742128000:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768111.751269:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768111.751296:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768111.751308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768111/real 1775768111] req@ffff90c203638380 x1862023742128128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768111.751318:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742128128/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768111.751324:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768111.751327:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768111.751329:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768111.751334:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128256:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768111.751346:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768111.751353:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768111/real 1775768111] req@ffff90c20363a680 x1862023742128256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768111.751361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742128256/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768111.751366:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768111.751367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768111.751369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128256:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768111.751373:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128384:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768111.751382:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768111.751388:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768111/real 1775768111] req@ffff90c20363b100 x1862023742128384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768111.751395:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742128384/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768111.751402:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768111.751404:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768111.751406:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128384:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768112.775031:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768113.799036:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768114.823043:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768115.847056:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768116.870984:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768116.870993:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768116.870997:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768116.871005:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4005 00000100:00080000:3.0:1775768116.871014:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768116.871034:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768116.871043:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768116.871067:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742128512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768116.871084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768116.871087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768116.871088:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768116.871090:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4005 00000100:00080000:3.0:1775768116.871092:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768116.871095:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768116.871102:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768116.871109:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742128640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768116.871114:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742128512:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768116.871120:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768116.871122:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768116.871123:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768116.871126:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4005 00000100:00080000:3.0:1775768116.871127:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768116.871129:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768116.871132:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768116.871138:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742128768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768116.871143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768116.871145:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768116.871146:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768116.871149:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4005 00000100:00080000:3.0:1775768116.871156:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768116.871158:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768116.871161:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768116.871165:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768116.871172:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742128896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768116.871194:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768116/real 1775768116] req@ffff90c218da3800 x1862023742128512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768116.871206:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742128512/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768116.871212:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768116.871215:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768116.871218:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742128512:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768116.871233:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768116.871245:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768116.871251:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768116/real 1775768116] req@ffff90c218da2d80 x1862023742128640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768116.871258:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742128640/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768116.871263:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768116.871264:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768116.871266:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768116.871269:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128768:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768116.871277:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768116.871282:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768116/real 1775768116] req@ffff90c218da0e00 x1862023742128768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768116.871292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742128768/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768116.871297:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768116.871298:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768116.871300:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128768:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768116.871303:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128896:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768116.871311:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768116.871315:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768116/real 1775768116] req@ffff90c218da0700 x1862023742128896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768116.871320:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742128896/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768116.871325:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768116.871327:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768116.871329:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742128896:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775768117.894994:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768118.919045:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768119.943028:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768120.967024:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768121.990957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768121.990965:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768121.990968:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768121.990976:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4010 00000100:00080000:3.0:1775768121.990984:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768121.990990:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768121.991012:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768121.991020:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742129024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768121.991038:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768121.991040:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768121.991042:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768121.991044:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4010 00000100:00080000:3.0:1775768121.991046:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768121.991048:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768121.991069:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742129024:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775768121.991071:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768121.991082:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742129152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768121.991094:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768121.991096:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768121.991097:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768121.991100:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4010 00000100:00080000:3.0:1775768121.991102:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768121.991104:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768121.991106:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768121.991112:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742129280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768121.991118:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768121.991120:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768121.991120:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768121.991123:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4010 00000100:00080000:3.0:1775768121.991139:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768121.991141:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768121.991143:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768121.991152:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768121.991153:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742129408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768121.991187:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffff90c218da1880 x1862023742129024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768121.991200:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742129024/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768121.991208:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768121.991212:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768121.991216:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742129024:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768121.991235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129152:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768121.991255:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768121.991266:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffff90c218da0380 x1862023742129152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768121.991274:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742129152/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768121.991279:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768121.991281:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768121.991283:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768121.991287:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129280:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768121.991299:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768121.991305:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffff90c218da0a80 x1862023742129280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768121.991311:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742129280/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768121.991315:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768121.991317:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768121.991319:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129280:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768121.991322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129408:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768121.991331:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768121.991336:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768121/real 1775768121] req@ffff90c218da3100 x1862023742129408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768121.991342:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742129408/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768146 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768121.991348:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768121.991349:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768121.991351:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129408:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768123.015066:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768124.038983:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768125.063039:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768126.087008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768127.110983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768127.110990:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768127.110993:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768127.111002:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4015 00010000:00010000:2.0:1775768127.111010:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768127.111021:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768127.111036:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768127.111070:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742129536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768127.111100:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768127.111103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768127.111104:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768127.111107:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4015 00000100:00080000:3.0:1775768127.111109:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768127.111112:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768127.111121:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768127.111133:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742129664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768127.111143:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768127.111146:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768127.111148:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768127.111153:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4015 00000100:00100000:2.0:1775768127.111155:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742129536:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768127.111157:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768127.111165:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768127.111183:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768127.111194:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742129792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768127.111200:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768127.111203:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768127.111204:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768127.111207:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4015 00000100:00080000:3.0:1775768127.111209:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768127.111214:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775768127.111215:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775768127.111218:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768127.111236:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742129920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768127.111255:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768127/real 1775768127] req@ffff90c218da0000 x1862023742129536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768127.111272:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742129536/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768127.111281:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768127.111285:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768127.111288:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742129536:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768127.111295:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768127.111317:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768127.111327:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768127/real 1775768127] req@ffff90c218da1f80 x1862023742129664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768127.111338:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742129664/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768127.111346:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768127.111349:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768127.111351:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768127.111372:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129792:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768127.111390:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768127.111399:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768127/real 1775768127] req@ffff90c218da3480 x1862023742129792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768127.111410:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742129792/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768127.111418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768127.111421:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768127.111423:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129792:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768127.111429:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129920:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768127.111444:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768127.111452:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768127/real 1775768127] req@ffff90c218da3100 x1862023742129920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768127.111469:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742129920/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768127.111478:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768127.111481:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768127.111484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742129920:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768128.134992:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768129.159032:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768130.182991:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768131.207021:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768132.230981:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768132.230991:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768132.230995:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768132.231004:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4020 00010000:00010000:1.0:1775768132.231014:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768132.231015:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768132.231020:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775768132.231053:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742130048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768132.231073:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768132.231075:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768132.231077:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768132.231079:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4020 00000100:00080000:0.0:1775768132.231081:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768132.231084:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768132.231091:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768132.231099:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742130176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768132.231105:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742130048:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775768132.231109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768132.231111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768132.231112:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768132.231115:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4020 00000100:00080000:0.0:1775768132.231117:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768132.231119:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768132.231122:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768132.231130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742130304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768132.231135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768132.231137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768132.231138:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768132.231140:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4020 00000100:00080000:0.0:1775768132.231143:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768132.231145:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768132.231147:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768132.231153:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768132.231156:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742130432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768132.231186:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768132/real 1775768132] req@ffff90c203639500 x1862023742130048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768132.231198:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742130048/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768132.231204:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768132.231207:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768132.231209:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742130048:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768132.231225:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130176:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768132.231237:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768132.231244:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768132/real 1775768132] req@ffff90c20363b480 x1862023742130176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768132.231250:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742130176/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768132.231259:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768132.231261:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768132.231263:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768132.231266:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130304:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768132.231274:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768132.231278:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768132/real 1775768132] req@ffff90c203639f80 x1862023742130304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768132.231284:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742130304/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768132.231288:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768132.231290:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768132.231291:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130304:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768132.231294:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130432:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768132.231302:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768132.231306:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768132/real 1775768132] req@ffff90c20363bb80 x1862023742130432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768132.231311:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742130432/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768132.231317:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768132.231318:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768132.231321:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130432:192.168.202.155@tcp:8:kworker.0 00010000:00010000:2.0:1775768133.255009:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768134.279024:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768135.303059:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768136.327034:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768137.351067:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768137.351078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768137.351087:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768137.351097:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4025 00000100:00080000:3.0:1775768137.351109:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768137.351116:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768137.351118:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768137.351154:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742130560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768137.351185:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768137.351189:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768137.351191:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768137.351195:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4025 00000100:00080000:3.0:1775768137.351200:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768137.351204:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768137.351214:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768137.351228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742130688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768137.351235:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742130560:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768137.351246:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768137.351250:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768137.351252:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768137.351257:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4025 00000100:00080000:3.0:1775768137.351261:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768137.351266:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768137.351272:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768137.351286:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742130816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768137.351296:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768137.351299:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768137.351300:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768137.351304:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4025 00000100:00080000:3.0:1775768137.351308:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768137.351312:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768137.351318:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768137.351324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768137.351337:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742130944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768137.351377:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768137/real 1775768137] req@ffff90c218da0a80 x1862023742130560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768137.351396:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742130560/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768137.351407:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768137.351412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768137.351416:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742130560:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768137.351443:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768137.351465:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768137.351477:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768137/real 1775768137] req@ffff90c218da0380 x1862023742130688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768137.351489:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742130688/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768137.351496:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768137.351499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768137.351502:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768137.351508:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130816:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768137.351527:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768137.351570:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768137/real 1775768137] req@ffff90c218da1880 x1862023742130816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768137.351584:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742130816/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768137.351592:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768137.351595:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768137.351598:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130816:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768137.351604:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130944:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768137.351623:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768137.351633:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768137/real 1775768137] req@ffff90c218da0700 x1862023742130944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768137.351643:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742130944/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768137.351654:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768137.351657:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768137.351660:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742130944:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768138.375072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768139.399059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768140.423102:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768141.447033:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768142.470973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768142.470983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768142.470988:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768142.470999:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4030 00010000:00010000:2.0:1775768142.471003:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768142.471011:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768142.471018:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768142.471059:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742131072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768142.471083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768142.471086:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768142.471088:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768142.471090:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4030 00000100:00080000:3.0:1775768142.471092:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768142.471094:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768142.471103:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768142.471115:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742131200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768142.471126:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775768142.471130:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742131072:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768142.471131:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768142.471135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768142.471142:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4030 00000100:00080000:3.0:1775768142.471146:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768142.471150:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768142.471156:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768142.471169:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742131328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768142.471178:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768142.471182:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768142.471183:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768142.471188:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4030 00000100:00080000:3.0:1775768142.471202:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768142.471207:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775768142.471213:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775768142.471213:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768142.471227:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742131456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768142.471260:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768142/real 1775768142] req@ffff90c218da0e00 x1862023742131072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768142.471279:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742131072/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768142.471290:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768142.471294:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768142.471299:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742131072:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768142.471326:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768142.471348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768142.471361:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768142/real 1775768142] req@ffff90c218da2d80 x1862023742131200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768142.471372:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742131200/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768142.471377:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768142.471379:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768142.471381:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768142.471385:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131328:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768142.471398:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768142.471404:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768142/real 1775768142] req@ffff90c218da3800 x1862023742131328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768142.471412:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742131328/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768142.471426:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768142.471428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768142.471431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131328:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768142.471437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131456:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768142.471455:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768142.471464:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768142/real 1775768142] req@ffff90c218da0700 x1862023742131456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768142.471473:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742131456/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768142.471480:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768142.471482:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768142.471484:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131456:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768143.494997:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768144.519041:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1775768145.542969:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768146.567060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768147.590978:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768147.590988:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768147.590993:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768147.591004:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4036 00000100:00080000:0.0:1775768147.591015:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768147.591023:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775768147.591051:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768147.591064:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742131584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768147.591097:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768147.591100:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768147.591102:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768147.591105:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4036 00000100:00080000:0.0:1775768147.591109:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768147.591112:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768147.591121:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768147.591135:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742131712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768147.591145:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768147.591149:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768147.591154:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775768147.591157:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742131584:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775768147.591163:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4036 00000100:00080000:0.0:1775768147.591168:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768147.591173:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768147.591179:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768147.591196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742131840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768147.591206:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768147.591210:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768147.591211:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768147.591215:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4036 00000100:00080000:0.0:1775768147.591220:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768147.591224:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768147.591231:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768147.591243:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768147.591249:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742131968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768147.591304:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768147/real 1775768147] req@ffff90c20363b800 x1862023742131584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768147.591323:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742131584/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768147.591331:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768147.591335:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768147.591339:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742131584:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768147.591368:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131712:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768147.591395:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768147.591407:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768147/real 1775768147] req@ffff90c20363bb80 x1862023742131712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768147.591418:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742131712/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768147.591425:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768147.591429:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768147.591432:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768147.591449:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131840:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768147.591470:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768147.591481:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768147/real 1775768147] req@ffff90c203639f80 x1862023742131840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768147.591492:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742131840/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768147.591497:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768147.591499:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768147.591501:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131840:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768147.591506:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131968:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768147.591520:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768147.591527:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768147/real 1775768147] req@ffff90c20363b480 x1862023742131968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768147.591534:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742131968/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768147.591541:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768147.591543:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768147.591545:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742131968:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768148.616827:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768149.639027:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768150.663035:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768151.687055:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768152.710983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768152.710992:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768152.710996:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768152.711005:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4041 00000100:00080000:3.0:1775768152.711015:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768152.711022:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768152.711032:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768152.711055:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742132096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768152.711081:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768152.711085:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768152.711086:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768152.711089:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4041 00000100:00080000:3.0:1775768152.711093:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768152.711097:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768152.711106:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768152.711115:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742132096:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768152.711119:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742132224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768152.711131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768152.711134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768152.711136:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768152.711140:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4041 00000100:00080000:3.0:1775768152.711143:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768152.711147:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768152.711151:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768152.711160:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742132352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768152.711167:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775768152.711174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768152.711178:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768152.711180:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768152.711185:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4041 00000100:00080000:3.0:1775768152.711188:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768152.711191:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768152.711197:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768152.711207:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768152/real 1775768152] req@ffff90c218da1880 x1862023742132096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1775768152.711209:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742132480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768152.711248:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742132096/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768152.711267:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768152.711272:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768152.711276:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742132096:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768152.711296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768152.711315:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768152.711334:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768152/real 1775768152] req@ffff90c218da0380 x1862023742132224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768152.711347:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742132224/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768152.711360:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768152.711362:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768152.711364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768152.711368:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132352:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768152.711381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768152.711403:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768152/real 1775768152] req@ffff90c218da0a80 x1862023742132352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768152.711411:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742132352/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768152.711416:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768152.711417:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768152.711419:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132352:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768152.711423:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132480:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768152.711438:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768152.711447:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768152/real 1775768152] req@ffff90c218da3100 x1862023742132480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768152.711457:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742132480/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768152.711466:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768152.711469:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768152.711479:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132480:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768153.735037:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768154.759072:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768155.783082:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768156.807055:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768157.831019:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768157.831029:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768157.831041:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768157.831047:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768157.831059:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4046 00000100:00080000:0.0:1775768157.831073:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768157.831082:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775768157.831130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742132608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768157.831164:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768157.831168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768157.831170:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768157.831175:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4046 00000100:00080000:0.0:1775768157.831181:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768157.831186:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768157.831199:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768157.831200:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742132608:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768157.831217:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742132736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768157.831230:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768157.831234:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768157.831235:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768157.831239:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4046 00000100:00080000:0.0:1775768157.831244:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768157.831247:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768157.831251:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768157.831262:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742132864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768157.831268:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768157.831271:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768157.831272:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768157.831274:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4046 00000100:00080000:0.0:1775768157.831276:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768157.831278:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768157.831282:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768157.831292:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742132992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768157.831318:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768157.831366:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768157.831383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132992:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768158.855175:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768159.879012:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768160.903134:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775768160.903250:0:2414:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775768160.903342:0:2414:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775768160.903365:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775768160.903410:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775768160.903422:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775768160.903433:0:2414:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775768160.903457:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768157/real 1775768160] req@ffff90c20363b100 x1862023742132736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768160.904013:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742132736/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768160.904029:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768160.904034:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768160.904039:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768160.904046:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768157/real 1775768160] req@ffff90c20363a680 x1862023742132864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768160.904057:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742132864/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768160.904064:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768160.904067:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768160.904070:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132864:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768160.904077:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768157/real 1775768160] req@ffff90c203638380 x1862023742132992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768160.904088:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742132992/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768160.904099:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768160.904102:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768160.904105:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742132992:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775768160.904114:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768157/real 1775768160] req@ffff90c203639500 x1862023742132608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768160.904125:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742132608/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768160.904133:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768160.904138:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768160.904141:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742132608:192.168.202.155@tcp:250:kworker.0 00010000:00010000:1.0:1775768161.927035:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768162.950987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768162.950995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768162.950998:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768162.951007:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4051 00010000:00010000:1.0:1775768162.951015:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768162.951017:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768162.951024:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768162.951054:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742133120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768162.951074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768162.951077:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768162.951078:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768162.951080:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4051 00000100:00080000:3.0:1775768162.951083:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768162.951087:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768162.951096:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768162.951111:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742133248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768162.951116:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742133120:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768162.951134:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768162.951137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768162.951139:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768162.951144:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4051 00000100:00080000:3.0:1775768162.951147:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768162.951151:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768162.951157:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768162.951173:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742133376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768162.951184:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768162.951187:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768162.951189:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1775768162.951196:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775768162.951196:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4051 00000100:00080000:3.0:1775768162.951202:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768162.951206:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768162.951212:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768162.951226:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742133504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768162.951247:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768162/real 1775768162] req@ffff90c218da3480 x1862023742133120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768162.951265:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742133120/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768162.951277:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768162.951281:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768162.951285:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742133120:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768162.951338:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768162.951361:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768162.951372:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768162/real 1775768162] req@ffff90c218da1f80 x1862023742133248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768162.951384:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742133248/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768162.951392:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768162.951396:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768162.951399:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768162.951404:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133376:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768162.951420:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768162.951428:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768162/real 1775768162] req@ffff90c218da0000 x1862023742133376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768162.951439:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742133376/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768162.951446:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768162.951450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768162.951453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133376:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768162.951459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133504:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768162.951476:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768162.951489:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768162/real 1775768162] req@ffff90c218da3100 x1862023742133504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768162.951514:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742133504/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768187 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768162.951525:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768162.951528:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768162.951531:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133504:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768163.975048:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768164.999016:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768166.023064:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768167.047011:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768168.070952:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768168.070960:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768168.070965:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768168.070973:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4056 00000100:00080000:3.0:1775768168.070982:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768168.070988:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768168.070989:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768168.071017:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742133632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768168.071036:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768168.071038:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768168.071039:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768168.071042:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4056 00000100:00080000:3.0:1775768168.071044:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768168.071045:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768168.071052:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768168.071060:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742133760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768168.071066:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768168.071070:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768168.071071:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775768168.071072:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742133632:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768168.071075:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4056 00000100:00080000:3.0:1775768168.071079:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768168.071081:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768168.071084:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768168.071091:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742133888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768168.071102:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768168.071105:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768168.071106:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768168.071110:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4056 00000100:00080000:3.0:1775768168.071120:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768168.071123:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775768168.071128:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775768168.071130:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768168.071149:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742134016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768168.071162:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768168/real 1775768168] req@ffff90c218da0a80 x1862023742133632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768168.071175:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742133632/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768168.071181:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768168.071184:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768168.071186:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742133632:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768168.071203:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768168.071218:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768168.071224:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768168/real 1775768168] req@ffff90c218da0380 x1862023742133760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768168.071231:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742133760/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768168.071258:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768168.071260:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768168.071263:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768168.071266:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133888:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768168.071279:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768168.071285:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768168/real 1775768168] req@ffff90c218da1880 x1862023742133888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768168.071292:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742133888/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768168.071296:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768168.071298:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768168.071300:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742133888:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768168.071303:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134016:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768168.071311:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768168.071316:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768168/real 1775768168] req@ffff90c218da0700 x1862023742134016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768168.071322:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742134016/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768168.071327:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768168.071329:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768168.071331:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134016:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768169.095008:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768170.119002:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768171.142975:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768172.166997:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768173.191039:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768173.191049:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768173.191055:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768173.191067:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4061 00010000:00010000:3.0:1775768173.191068:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768173.191080:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768173.191087:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768173.191129:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742134144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768173.191151:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768173.191156:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768173.191158:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768173.191160:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4061 00000100:00080000:1.0:1775768173.191162:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768173.191165:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768173.191174:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768173.191201:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742134272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768173.191208:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768173.191211:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768173.191212:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768173.191217:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4061 00000100:00100000:2.0:1775768173.191217:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742134144:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775768173.191222:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768173.191226:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768173.191230:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768173.191239:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742134400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768173.191245:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768173.191248:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768173.191249:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768173.191251:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4061 00000100:00080000:1.0:1775768173.191254:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768173.191256:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768173.191260:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768173.191268:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742134528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768173.191297:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768173.191340:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768173/real 1775768173] req@ffff90c203495c00 x1862023742134144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768173.191361:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742134144/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768173.191371:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768173.191376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768173.191380:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742134144:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768173.191404:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134272:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768173.191421:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768173.191429:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768173/real 1775768173] req@ffff90c203495880 x1862023742134272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768173.191437:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742134272/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768173.191442:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768173.191444:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768173.191447:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768173.191450:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134400:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768173.191460:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768173.191466:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768173/real 1775768173] req@ffff90c203497100 x1862023742134400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768173.191473:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742134400/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768173.191477:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768173.191479:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768173.191480:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134400:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768173.191483:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134528:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768173.191494:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768173.191502:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768173/real 1775768173] req@ffff90c203496d80 x1862023742134528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768173.191513:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742134528/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768173.191523:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768173.191526:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768173.191529:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134528:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768174.215081:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768175.239087:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768176.263078:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768177.287083:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768178.311007:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768178.311019:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768178.311026:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768178.311039:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4066 00000100:00080000:3.0:1775768178.311054:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:1.0:1775768178.311058:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768178.311063:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768178.311116:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742134656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768178.311163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768178.311168:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768178.311170:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768178.311175:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4066 00000100:00080000:3.0:1775768178.311180:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768178.311185:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768178.311197:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768178.311206:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742134656:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768178.311214:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742134784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768178.311227:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768178.311231:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768178.311233:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768178.311238:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4066 00000100:00080000:3.0:1775768178.311242:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768178.311246:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768178.311252:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768178.311265:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742134912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768178.311275:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768178.311279:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768178.311280:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768178.311284:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4066 00000100:00080000:3.0:1775768178.311290:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775768178.311291:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775768178.311298:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768178.311305:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768178.311320:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742135040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768178.311349:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768178/real 1775768178] req@ffff90c218da3800 x1862023742134656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768178.311370:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742134656/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768178.311392:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768178.311398:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768178.311403:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742134656:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768178.311432:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768178.311455:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768178.311467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768178/real 1775768178] req@ffff90c218da2d80 x1862023742134784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768178.311481:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742134784/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768178.311489:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768178.311493:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768178.311496:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768178.311502:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134912:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768178.311520:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768178.311530:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768178/real 1775768178] req@ffff90c218da0e00 x1862023742134912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768178.311542:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742134912/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768178.311550:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768178.311554:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768178.311557:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742134912:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768178.311563:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135040:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768178.311582:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768178.311591:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768178/real 1775768178] req@ffff90c218da0700 x1862023742135040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768178.311602:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742135040/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768178.311613:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768178.311616:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768178.311619:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135040:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768179.335085:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768180.359043:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768181.383098:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768182.407022:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768183.431022:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768183.431032:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768183.431037:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768183.431049:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4071 00000100:00080000:0.0:1775768183.431063:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768183.431072:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768183.431081:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768183.431142:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742135168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768183.431167:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768183.431170:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768183.431172:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768183.431175:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4071 00000100:00080000:0.0:1775768183.431178:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768183.431181:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768183.431190:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768183.431200:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742135296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768183.431209:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768183.431213:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768183.431214:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768183.431219:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4071 00000100:00080000:0.0:1775768183.431222:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775768183.431222:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742135168:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775768183.431227:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768183.431233:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768183.431239:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742135424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768183.431247:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768183.431249:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768183.431251:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768183.431253:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4071 00000100:00080000:0.0:1775768183.431256:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768183.431265:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768183.431268:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768183.431281:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742135552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768183.431320:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768183.431362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768183/real 1775768183] req@ffff90c203638000 x1862023742135168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768183.431379:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742135168/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768183.431388:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768183.431392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768183.431395:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742135168:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768183.431417:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768183.431434:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768183.431441:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768183/real 1775768183] req@ffff90c203639180 x1862023742135296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768183.431451:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742135296/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768183.431456:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768183.431459:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768183.431461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768183.431464:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135424:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768183.431474:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768183.431479:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768183/real 1775768183] req@ffff90c20363aa00 x1862023742135424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768183.431486:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742135424/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768183.431491:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768183.431492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768183.431494:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135424:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768183.431498:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135552:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768183.431507:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768183.431512:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768183/real 1775768183] req@ffff90c203638e00 x1862023742135552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768183.431519:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742135552/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768183.431530:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768183.431533:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768183.431535:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135552:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768184.455007:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768185.479149:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768186.503120:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768187.527092:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768188.550992:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768188.551005:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768188.551011:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00010000:00010000:3.0:1775768188.551020:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768188.551024:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4077 00000100:00080000:1.0:1775768188.551039:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768188.551062:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768188.551109:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742135680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768188.551137:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768188.551141:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768188.551143:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768188.551155:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4077 00000100:00080000:1.0:1775768188.551159:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768188.551163:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768188.551175:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768188.551175:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742135680:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775768188.551193:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742135808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768188.551204:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768188.551207:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768188.551209:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768188.551213:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4077 00000100:00080000:1.0:1775768188.551216:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768188.551220:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768188.551225:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768188.551236:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742135936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768188.551245:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768188.551248:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768188.551250:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768188.551253:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4077 00000100:00080000:1.0:1775768188.551257:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775768188.551259:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775768188.551269:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768188.551281:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768188.551295:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742136064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768188.551301:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768188/real 1775768188] req@ffff90c203494e00 x1862023742135680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768188.551322:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742135680/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768188.551330:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768188.551334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768188.551336:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742135680:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768188.551357:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135808:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768188.551370:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768188.551377:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768188/real 1775768188] req@ffff90c203495f80 x1862023742135808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768188.551385:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742135808/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768188.551390:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768188.551392:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768188.551394:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768188.551398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135936:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768188.551407:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768188.551412:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768188/real 1775768188] req@ffff90c203494380 x1862023742135936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768188.551419:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742135936/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768188.551423:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768188.551425:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768188.551427:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742135936:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768188.551429:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136064:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768188.551443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768188.551448:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768188/real 1775768188] req@ffff90c203494000 x1862023742136064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768188.551453:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742136064/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768188.551459:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768188.551461:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768188.551463:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136064:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768189.575069:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768190.599094:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768191.623061:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768192.647046:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768193.671024:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768193.671037:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768193.671043:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768193.671055:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4082 00010000:00010000:3.0:1775768193.671061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768193.671071:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768193.671080:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768193.671130:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742136192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768193.671162:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768193.671166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768193.671168:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768193.671174:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4082 00000100:00080000:1.0:1775768193.671179:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768193.671183:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768193.671195:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768193.671203:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742136192:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775768193.671216:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742136320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768193.671231:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768193.671235:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768193.671237:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768193.671242:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4082 00000100:00080000:1.0:1775768193.671249:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768193.671253:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775768193.671267:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:1.0:1775768193.671269:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768193.671291:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742136448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768193.671302:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768193/real 1775768193] req@ffff90c203497480 x1862023742136192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768193.671305:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768193.671318:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768193.671325:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768193.671334:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4082 00000100:00100000:2.0:1775768193.671339:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742136192/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1775768193.671344:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768193.671350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1775768193.671354:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00080000:2.0:1775768193.671360:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000008:00100000:1.0:1775768193.671367:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768193.671367:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742136192:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775768193.671393:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742136576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768193.671400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768193.671443:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768193.671458:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768193/real 1775768193] req@ffff90c203494000 x1862023742136320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768193.671470:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742136320/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768193.671479:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768193.671482:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768193.671486:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768193.671491:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136448:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768193.671509:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768193.671518:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768193/real 1775768193] req@ffff90c203494380 x1862023742136448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768193.671530:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742136448/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768193.671544:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768193.671547:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768193.671550:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136448:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768193.671563:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136576:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768193.671578:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768193.671584:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768193/real 1775768193] req@ffff90c203495f80 x1862023742136576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768193.671591:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742136576/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768193.671597:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768193.671599:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768193.671601:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136576:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768194.695089:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768195.719062:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768196.743056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768197.767030:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768198.790986:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00010000:00010000:1.0:1775768198.790994:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768198.790995:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768198.791000:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768198.791010:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4087 00000100:00080000:3.0:1775768198.791019:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768198.791025:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768198.791057:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742136704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768198.791079:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768198.791083:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768198.791084:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768198.791088:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4087 00000100:00080000:3.0:1775768198.791092:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768198.791096:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768198.791104:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742136704:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775768198.791105:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768198.791119:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742136832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768198.791129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768198.791132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768198.791134:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768198.791137:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4087 00000100:00080000:3.0:1775768198.791147:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768198.791150:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000800:00000100:2.0:1775768198.791155:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000008:00100000:3.0:1775768198.791155:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768198.791165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742136960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768198.791174:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768198.791177:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768198.791179:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768198.791183:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4087 00000100:00100000:2.0:1775768198.791185:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768198/real 1775768198] req@ffff90c218da1880 x1862023742136704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768198.791187:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768198.791194:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768198.791200:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768198.791203:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742136704/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768198.791211:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768198.791216:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:3.0:1775768198.791216:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742137088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768198.791222:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742136704:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768198.791246:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136832:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768198.791261:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768198.791267:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768198/real 1775768198] req@ffff90c218da0380 x1862023742136832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768198.791274:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742136832/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768198.791279:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768198.791281:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768198.791283:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768198.791285:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136960:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768198.791294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768198.791298:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768198/real 1775768198] req@ffff90c218da0a80 x1862023742136960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768198.791304:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742136960/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768198.791308:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768198.791310:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768198.791311:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742136960:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768198.791314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137088:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768198.791323:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768198.791327:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768198/real 1775768198] req@ffff90c218da3100 x1862023742137088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768198.791333:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742137088/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768198.791339:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768198.791340:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768198.791341:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137088:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768199.815104:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768200.839037:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768201.863007:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768202.887116:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768203.910939:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768203.910946:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768203.910950:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768203.910958:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4092 00000100:00080000:3.0:1775768203.910968:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768203.910975:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775768203.911000:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768203.911015:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742137216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768203.911036:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768203.911039:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768203.911040:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768203.911043:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4092 00000100:00080000:3.0:1775768203.911046:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768203.911048:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768203.911056:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768203.911064:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742137216:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768203.911067:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742137344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768203.911076:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768203.911078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768203.911080:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768203.911083:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4092 00000100:00080000:3.0:1775768203.911085:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768203.911087:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768203.911090:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768203.911098:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742137472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768203.911107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768203.911110:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768203.911111:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768203.911115:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4092 00000800:00000100:2.0:1775768203.911120:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775768203.911122:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768203.911127:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768203.911133:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768203.911145:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742137600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768203.911157:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768203/real 1775768203] req@ffff90c218da0000 x1862023742137216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768203.911175:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742137216/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768203.911183:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768203.911185:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768203.911187:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742137216:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768203.911204:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137344:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768203.911218:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768203.911225:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768203/real 1775768203] req@ffff90c218da1f80 x1862023742137344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768203.911232:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742137344/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768203.911240:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768203.911242:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768203.911244:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137344:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768203.911247:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137472:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768203.911255:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768203.911260:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768203/real 1775768203] req@ffff90c218da3480 x1862023742137472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768203.911265:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742137472/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768203.911270:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768203.911271:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768203.911273:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137472:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768203.911275:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137600:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768203.911283:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768203.911287:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768203/real 1775768203] req@ffff90c218da3100 x1862023742137600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768203.911293:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742137600/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768203.911298:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768203.911299:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768203.911301:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137600:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768204.935029:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768205.959024:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768206.983047:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768208.007087:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768209.030977:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768209.030987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768209.030992:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768209.031002:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4097 00000100:00080000:3.0:1775768209.031013:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768209.031021:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768209.031061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768209.031063:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742137728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768209.031099:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768209.031103:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768209.031104:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768209.031108:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4097 00000100:00080000:3.0:1775768209.031111:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768209.031114:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768209.031123:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768209.031136:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742137728:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768209.031136:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742137856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768209.031151:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768209.031153:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768209.031154:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768209.031158:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4097 00000100:00080000:3.0:1775768209.031161:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768209.031162:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768209.031166:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768209.031175:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742137984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768209.031181:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768209.031183:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768209.031185:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768209.031188:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4097 00000100:00080000:3.0:1775768209.031191:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768209.031193:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768209.031197:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768209.031198:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768209.031210:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742138112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768209.031235:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768209/real 1775768209] req@ffff90c218da0a80 x1862023742137728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768209.031256:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742137728/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768209.031273:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768209.031277:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768209.031281:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742137728:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768209.031304:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137856:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768209.031326:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768209.031338:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768209/real 1775768209] req@ffff90c218da0380 x1862023742137856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768209.031350:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742137856/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768209.031358:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768209.031361:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768209.031365:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137856:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768209.031371:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137984:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768209.031393:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768209.031405:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768209/real 1775768209] req@ffff90c218da1880 x1862023742137984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768209.031416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742137984/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768209.031424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768209.031428:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768209.031431:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742137984:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768209.031437:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138112:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768209.031458:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768209.031481:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768209/real 1775768209] req@ffff90c218da0700 x1862023742138112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768209.031494:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742138112/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768209.031503:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768209.031507:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768209.031510:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138112:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775768210.055044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768211.079025:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768212.103059:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768213.127026:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768214.150997:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768214.151007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768214.151011:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768214.151021:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4102 00000100:00080000:3.0:1775768214.151033:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768214.151040:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768214.151081:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768214.151088:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742138240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768214.151129:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768214.151132:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768214.151133:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768214.151136:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4102 00000100:00080000:3.0:1775768214.151139:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768214.151142:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768214.151152:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768214.151162:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742138368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768214.151172:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775768214.151175:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742138240:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768214.151176:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768214.151181:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768214.151192:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4102 00000100:00080000:3.0:1775768214.151194:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768214.151196:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768214.151199:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768214.151208:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742138496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768214.151214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768214.151217:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768214.151219:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768214.151222:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4102 00000100:00080000:3.0:1775768214.151241:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775768214.151249:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775768214.151249:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768214.151257:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768214.151275:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742138624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768214.151285:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768214/real 1775768214] req@ffff90c218da0e00 x1862023742138240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768214.151302:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742138240/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768214.151311:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768214.151315:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768214.151318:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742138240:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768214.151336:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138368:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768214.151348:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768214.151355:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768214/real 1775768214] req@ffff90c218da2d80 x1862023742138368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768214.151362:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742138368/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768214.151367:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768214.151369:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768214.151371:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138368:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768214.151374:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138496:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768214.151383:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768214.151387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768214/real 1775768214] req@ffff90c218da3800 x1862023742138496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768214.151393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742138496/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768214.151397:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768214.151399:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768214.151401:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138496:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768214.151404:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138624:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768214.151413:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768214.151418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768214/real 1775768214] req@ffff90c218da0700 x1862023742138624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768214.151424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742138624/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768214.151429:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768214.151431:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768214.151432:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138624:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768215.175086:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768216.199082:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768217.223015:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768218.247049:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768219.270950:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768219.270960:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768219.270964:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768219.270972:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4107 00000100:00080000:3.0:1775768219.270982:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768219.270988:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775768219.271006:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768219.271022:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742138752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768219.271049:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768219.271052:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768219.271054:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768219.271057:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4107 00000100:00080000:3.0:1775768219.271060:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768219.271063:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768219.271070:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768219.271079:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742138752:192.168.202.155@tcp:250:kworker.0 00000100:00100000:3.0:1775768219.271080:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742138880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768219.271094:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768219.271097:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768219.271099:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768219.271104:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4107 00000100:00080000:3.0:1775768219.271107:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768219.271111:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768219.271116:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768219.271127:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742139008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768219.271135:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768219.271137:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768219.271138:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768219.271143:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4107 00000800:00000100:2.0:1775768219.271144:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:3.0:1775768219.271148:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768219.271152:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768219.271157:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768219.271165:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742139136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768219.271178:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768219/real 1775768219] req@ffff90c218da1880 x1862023742138752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768219.271190:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742138752/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768219.271198:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768219.271200:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768219.271203:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742138752:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768219.271219:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138880:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768219.271232:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768219.271238:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768219/real 1775768219] req@ffff90c218da0380 x1862023742138880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768219.271246:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742138880/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768219.271250:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768219.271252:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768219.271254:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742138880:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768219.271257:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139008:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768219.271265:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768219.271270:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768219/real 1775768219] req@ffff90c218da0a80 x1862023742139008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768219.271276:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742139008/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768219.271280:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768219.271282:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768219.271284:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139008:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768219.271286:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139136:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768219.271295:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768219.271299:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768219/real 1775768219] req@ffff90c218da3100 x1862023742139136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768219.271304:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742139136/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768219.271313:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768219.271315:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768219.271317:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139136:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768220.295094:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768221.319075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768222.343102:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768223.367016:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768224.391020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768224.391034:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768224.391039:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768224.391051:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4112 00010000:00010000:3.0:1775768224.391064:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768224.391067:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768224.391076:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768224.391133:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742139264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768224.391162:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768224.391166:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768224.391168:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768224.391172:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4112 00000100:00080000:1.0:1775768224.391176:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768224.391181:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768224.391194:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768224.391204:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742139264:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775768224.391210:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742139392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768224.391222:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768224.391226:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768224.391227:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768224.391232:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4112 00000100:00080000:1.0:1775768224.391236:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768224.391239:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768224.391245:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768224.391259:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742139520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768224.391269:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768224.391273:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768224.391275:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768224.391280:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4112 00000100:00080000:1.0:1775768224.391284:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768224.391288:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768224.391294:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768224.391308:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742139648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768224.391328:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768224.391359:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768224.391383:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139648:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775768225.414974:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768226.439012:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768227.463075:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1775768227.463150:0:2415:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:0.0:1775768227.463182:0:2415:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1775768227.463195:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775768227.463229:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775768227.463237:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:0.0:1775768227.463248:0:2415:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:2.0:1775768227.463368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768224/real 1775768227] req@ffff90c203496d80 x1862023742139392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768227.463393:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742139392/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768227.463406:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768227.463412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768227.463417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139392:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768227.463429:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768224/real 1775768227] req@ffff90c203497100 x1862023742139520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768227.463439:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742139520/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768227.463445:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768227.463449:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768227.463451:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139520:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768227.463459:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768224/real 1775768227] req@ffff90c203495880 x1862023742139648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768227.463471:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742139648/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768227.463481:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768227.463485:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768227.463488:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139648:192.168.202.155@tcp:8:kworker.0 00000100:00100000:2.0:1775768227.463494:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768224/real 1775768227] req@ffff90c203494e00 x1862023742139264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768227.463507:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742139264/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768227.463514:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768227.463517:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768227.463592:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742139264:192.168.202.155@tcp:250:kworker.0 00010000:00010000:0.0:1775768228.487040:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768229.510979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768229.510997:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768229.511001:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768229.511014:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4117 00000100:00080000:1.0:1775768229.511026:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775768229.511026:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768229.511034:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768229.511068:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742139776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768229.511092:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768229.511095:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768229.511097:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768229.511101:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4117 00000100:00080000:1.0:1775768229.511105:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768229.511108:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768229.511117:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768229.511130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742139904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768229.511135:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742139776:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775768229.511152:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768229.511155:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768229.511157:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768229.511161:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4117 00000100:00080000:1.0:1775768229.511165:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768229.511169:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768229.511175:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768229.511185:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742140032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768229.511194:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768229.511197:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768229.511199:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768229.511204:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4117 00000800:00000100:2.0:1775768229.511209:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775768229.511210:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768229.511216:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768229.511221:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768229.511233:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742140160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768229.511250:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768229/real 1775768229] req@ffff90c203495c00 x1862023742139776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768229.511269:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742139776/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768229.511281:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768229.511286:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768229.511289:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742139776:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768229.511314:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139904:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768229.511334:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768229.511345:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768229/real 1775768229] req@ffff90c203497b80 x1862023742139904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768229.511357:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742139904/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768229.511363:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768229.511367:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768229.511369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742139904:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768229.511374:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140032:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768229.511392:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768229.511401:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768229/real 1775768229] req@ffff90c203496300 x1862023742140032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768229.511412:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742140032/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768229.511418:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768229.511421:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768229.511424:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140032:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768229.511430:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140160:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768229.511446:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768229.511454:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768229/real 1775768229] req@ffff90c203496a00 x1862023742140160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768229.511463:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742140160/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768229.511471:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768229.511474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768229.511477:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140160:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768230.535038:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768231.558993:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768232.582950:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768233.607026:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768234.631033:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768234.631065:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768234.631070:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768234.631074:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768234.631082:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4123 00000100:00080000:1.0:1775768234.631092:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768234.631097:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768234.631129:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742140288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768234.631182:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742140288:192.168.202.155@tcp:250:kworker.0 00000800:00000100:2.0:1775768234.631228:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775768234.631251:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768234.631254:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768234.631255:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768234.631258:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4123 00000100:00080000:1.0:1775768234.631260:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768234.631262:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768234.631269:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768234.631279:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742140416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768234.631333:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768234/real 1775768234] req@ffff90c203496680 x1862023742140288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768234.631346:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742140288/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768234.631352:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768234.631355:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768234.631357:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742140288:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768234.631373:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140416:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768234.631388:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768234.631395:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768234/real 1775768234] req@ffff90c203494a80 x1862023742140416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768234.631402:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742140416/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768234.631407:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768234.631408:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768234.631410:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140416:192.168.202.155@tcp:38:kworker.0 00000100:00080000:1.0:1775768234.631444:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768234.631447:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768234.631449:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768234.631454:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4123 00000100:00080000:1.0:1775768234.631457:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768234.631460:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768234.631464:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768234.631473:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742140544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768234.631481:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775768234.631490:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140544:192.168.202.155@tcp:38:kworker.0 00000100:00080000:1.0:1775768234.631496:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768234.631505:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768234.631514:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4123 00000100:00080000:1.0:1775768234.631518:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775768234.631520:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775768234.631525:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768234.631531:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768234.631533:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768234/real 1775768234] req@ffff90c203494700 x1862023742140544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768234.631546:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742140544/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768234.631547:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742140672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768234.631570:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768234.631575:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768234.631579:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140544:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768234.631605:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140672:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768234.631622:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768234.631629:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768234/real 1775768234] req@ffff90c203495500 x1862023742140672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768234.631636:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742140672/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768234.631654:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768234.631656:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768234.631658:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140672:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768235.655054:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768236.679019:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768237.703059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768238.727073:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768239.751017:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768239.751028:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768239.751034:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768239.751047:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4128 00000100:00080000:1.0:1775768239.751061:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768239.751070:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775768239.751095:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775768239.751121:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742140800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768239.751146:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768239.751149:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768239.751150:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768239.751153:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4128 00000100:00080000:1.0:1775768239.751157:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768239.751160:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768239.751169:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768239.751182:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742140928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768239.751187:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742140800:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775768239.751192:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768239.751195:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768239.751197:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768239.751200:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4128 00000100:00080000:1.0:1775768239.751201:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768239.751203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768239.751206:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768239.751212:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742141056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768239.751220:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768239.751224:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768239.751225:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768239.751229:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4128 00000100:00080000:1.0:1775768239.751233:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768239.751236:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768239.751241:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768239.751252:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742141184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768239.751271:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768239.751315:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768239/real 1775768239] req@ffff90c203495180 x1862023742140800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768239.751341:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742140800/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768239.751352:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768239.751356:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768239.751360:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742140800:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768239.751386:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140928:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768239.751407:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768239.751426:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768239/real 1775768239] req@ffff90c203495500 x1862023742140928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768239.751438:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742140928/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768239.751446:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768239.751450:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768239.751453:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742140928:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768239.751459:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141056:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768239.751479:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768239.751489:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768239/real 1775768239] req@ffff90c203494700 x1862023742141056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768239.751500:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742141056/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768239.751507:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768239.751510:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768239.751513:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141056:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768239.751519:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141184:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768239.751535:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768239.751543:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768239/real 1775768239] req@ffff90c203494a80 x1862023742141184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768239.751599:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742141184/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768239.751607:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768239.751610:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768239.751612:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141184:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775768240.775111:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768241.799094:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768242.823055:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768243.847068:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768244.870961:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768244.870971:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768244.870976:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768244.870985:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4133 00000100:00080000:1.0:1775768244.870997:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768244.871003:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775768244.871006:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775768244.871035:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742141312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768244.871055:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768244.871057:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768244.871059:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768244.871061:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4133 00000100:00080000:1.0:1775768244.871063:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768244.871066:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768244.871074:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768244.871082:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742141440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768244.871091:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768244.871096:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768244.871097:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1775768244.871098:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742141312:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775768244.871102:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4133 00000100:00080000:1.0:1775768244.871115:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768244.871120:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768244.871124:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768244.871132:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742141568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768244.871139:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768244.871143:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768244.871145:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768244.871150:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4133 00000100:00080000:1.0:1775768244.871153:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768244.871157:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768244.871162:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768244.871172:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742141696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768244.871192:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768244.871236:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768244/real 1775768244] req@ffff90c203496680 x1862023742141312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768244.871252:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742141312/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768244.871270:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768244.871274:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768244.871277:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742141312:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768244.871300:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141440:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768244.871320:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768244.871332:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768244/real 1775768244] req@ffff90c203496a00 x1862023742141440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768244.871343:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742141440/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768244.871350:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768244.871353:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768244.871355:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141440:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768244.871361:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141568:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768244.871378:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768244.871388:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768244/real 1775768244] req@ffff90c203496300 x1862023742141568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768244.871399:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742141568/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768244.871411:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768244.871414:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768244.871417:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141568:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768244.871423:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141696:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768244.871442:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768244.871451:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768244/real 1775768244] req@ffff90c203497b80 x1862023742141696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768244.871461:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742141696/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768244.871471:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768244.871474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768244.871477:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141696:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768245.895015:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768246.919027:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768247.943125:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768248.967128:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768249.990957:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768249.990971:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768249.990975:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768249.990982:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4138 00000100:00080000:1.0:1775768249.990993:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768249.991000:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768249.991037:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742141824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775768249.991039:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768249.991062:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768249.991065:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768249.991066:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768249.991069:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4138 00000100:00080000:1.0:1775768249.991072:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768249.991075:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768249.991083:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768249.991090:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742141824:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775768249.991094:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742141952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768249.991102:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768249.991104:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768249.991105:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768249.991108:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4138 00000100:00080000:1.0:1775768249.991109:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768249.991111:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768249.991114:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768249.991121:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742142080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768249.991132:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768249.991134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768249.991135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768249.991137:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4138 00000100:00080000:1.0:1775768249.991140:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775768249.991143:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:1.0:1775768249.991145:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768249.991149:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768249.991157:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742142208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768249.991176:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffff90c203495c00 x1862023742141824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768249.991189:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742141824/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768249.991195:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768249.991198:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768249.991201:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742141824:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768249.991217:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141952:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768249.991229:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768249.991235:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffff90c203494e00 x1862023742141952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768249.991242:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742141952/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768249.991247:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768249.991249:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768249.991251:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742141952:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768249.991254:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142080:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768249.991262:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768249.991266:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffff90c203495880 x1862023742142080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768249.991272:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742142080/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768249.991277:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768249.991278:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768249.991280:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142080:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768249.991283:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142208:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768249.991291:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768249.991295:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768249/real 1775768249] req@ffff90c203497100 x1862023742142208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768249.991302:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742142208/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768274 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768249.991308:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768249.991309:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768249.991311:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142208:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768251.015071:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768252.039023:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768253.063039:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768254.087022:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768255.111008:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768255.111016:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768255.111019:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768255.111027:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4143 00000100:00080000:2.0:1775768255.111035:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768255.111047:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768255.111081:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023742142336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768255.111107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768255.111109:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768255.111111:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768255.111115:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4143 00000100:00080000:2.0:1775768255.111119:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768255.111123:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768255.111133:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768255.111146:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023742142464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768255.111151:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742142336:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775768255.111168:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768255.111171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768255.111172:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768255.111176:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4143 00000100:00080000:2.0:1775768255.111179:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768255.111181:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768255.111186:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768255.111197:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023742142592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768255.111205:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775768255.111209:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768255.111212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768255.111214:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768255.111220:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4143 00000100:00080000:2.0:1775768255.111224:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768255.111228:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768255.111233:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768255.111237:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768255/real 1775768255] req@ffff90c2034be300 x1862023742142336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768255.111246:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023742142720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768255.111253:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023742142336/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768255.111267:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768255.111272:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768255.111277:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742142336:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768255.111302:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142464:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768255.111324:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768255.111334:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768255/real 1775768255] req@ffff90c2034bf480 x1862023742142464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1775768255.111341:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768255.111344:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023742142464/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768255.111354:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768255.111358:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768255.111362:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142464:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768255.111367:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142592:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768255.111386:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768255.111396:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768255/real 1775768255] req@ffff90c2034bce00 x1862023742142592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768255.111427:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023742142592/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768255.111432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768255.111434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768255.111436:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142592:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768255.111440:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142720:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768255.111456:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768255.111465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768255/real 1775768255] req@ffff90c2034bc380 x1862023742142720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768255.111476:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023742142720/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768255.111485:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768255.111488:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768255.111490:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142720:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775768256.135040:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768257.159085:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768258.183056:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768259.207140:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768260.231025:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768260.231038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768260.231044:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768260.231056:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4148 00000100:00080000:2.0:1775768260.231071:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768260.231078:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768260.231134:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023742142848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768260.231167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768260.231169:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768260.231171:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768260.231174:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4148 00000100:00080000:2.0:1775768260.231177:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768260.231180:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768260.231190:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768260.231202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023742142976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768260.231210:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768260.231212:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768260.231214:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768260.231216:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4148 00000100:00080000:2.0:1775768260.231218:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768260.231220:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768260.231222:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768260.231228:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023742143104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768260.231236:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768260.231238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768260.231239:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768260.231243:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4148 00000100:00080000:2.0:1775768260.231246:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768260.231252:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775768260.231253:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742142848:192.168.202.155@tcp:250:kworker.0 00000008:00100000:2.0:1775768260.231256:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768260.231266:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023742143232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768260.231336:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00010000:00010000:3.0:1775768260.231390:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768260.231399:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768260/real 1775768260] req@ffff90c2034bd180 x1862023742142848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768260.231416:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023742142848/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768260.231424:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768260.231427:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768260.231429:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742142848:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768260.231435:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142976:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768260.231453:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768260.231461:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768260/real 1775768260] req@ffff90c2034bdf80 x1862023742142976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768260.231468:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023742142976/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768260.231472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768260.231474:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768260.231476:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742142976:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768260.231498:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143104:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768260.231516:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768260.231525:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768260/real 1775768260] req@ffff90c2034bf100 x1862023742143104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768260.231542:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023742143104/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768260.231550:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768260.231552:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768260.231555:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143104:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768260.231561:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143232:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768260.231579:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768260.231590:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768260/real 1775768260] req@ffff90c2034bc700 x1862023742143232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768260.231601:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023742143232/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768260.231611:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768260.231613:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768260.231615:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143232:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768261.255080:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768262.278967:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768263.303048:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768264.327055:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768265.350995:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768265.351005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768265.351010:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768265.351019:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4153 00000100:00080000:1.0:1775768265.351030:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768265.351037:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:0.0:1775768265.351070:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775768265.351077:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742143360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768265.351104:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768265.351107:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768265.351109:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768265.351113:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4153 00000100:00080000:1.0:1775768265.351118:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768265.351122:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768265.351132:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768265.351137:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742143360:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775768265.351146:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742143488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768265.351157:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768265.351161:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768265.351162:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768265.351166:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4153 00000100:00080000:1.0:1775768265.351170:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768265.351173:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768265.351179:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1775768265.351202:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:1.0:1775768265.351203:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742143616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768265.351214:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768265.351218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768265.351220:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768265.351224:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4153 00000100:00080000:1.0:1775768265.351228:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768265.351232:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768265.351237:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768265.351241:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768265/real 1775768265] req@ffff90c203496d80 x1862023742143360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1775768265.351257:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742143744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768265.351260:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742143360/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768265.351290:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768265.351295:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768265.351322:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742143360:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768265.351341:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143488:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768265.351359:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768265.351368:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768265/real 1775768265] req@ffff90c203495f80 x1862023742143488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768265.351377:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742143488/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768265.351383:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768265.351393:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768265.351396:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143488:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768265.351400:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143616:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768265.351414:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768265.351421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768265/real 1775768265] req@ffff90c203494380 x1862023742143616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768265.351430:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742143616/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768265.351435:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768265.351438:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768265.351440:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143616:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768265.351445:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143744:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768265.351458:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768265.351465:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768265/real 1775768265] req@ffff90c203494000 x1862023742143744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768265.351473:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742143744/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768265.351481:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768265.351483:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768265.351486:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742143744:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768266.375057:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768267.399060:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768268.423068:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768269.447080:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768270.470986:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768270.471000:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768270.471005:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768270.471016:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4158 00000100:00080000:0.0:1775768270.471030:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768270.471036:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775768270.471044:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768270.471079:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638700 x1862023742143872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768270.471103:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768270.471106:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768270.471108:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768270.471111:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4158 00000100:00080000:0.0:1775768270.471113:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768270.471117:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768270.471126:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768270.471135:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742143872:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768270.471138:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638e00 x1862023742144000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768270.471147:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768270.471150:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768270.471151:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768270.471153:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4158 00000100:00080000:0.0:1775768270.471155:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768270.471157:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768270.471160:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768270.471167:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363aa00 x1862023742144128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768270.471173:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768270.471175:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768270.471176:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768270.471179:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4158 00000100:00080000:0.0:1775768270.471181:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768270.471183:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768270.471186:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768270.471192:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639180 x1862023742144256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768270.471203:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768270.471257:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768270/real 1775768270] req@ffff90c203638700 x1862023742143872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768270.471278:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638700 x1862023742143872/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768270.471287:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768270.471290:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768270.471293:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742143872:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768270.471318:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144000:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768270.471340:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768270.471352:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768270/real 1775768270] req@ffff90c203638e00 x1862023742144000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768270.471364:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638e00 x1862023742144000/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768270.471372:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768270.471376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768270.471379:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144000:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768270.471386:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144128:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768270.471412:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768270.471420:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768270/real 1775768270] req@ffff90c20363aa00 x1862023742144128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768270.471427:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363aa00 x1862023742144128/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768270.471432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768270.471434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768270.471436:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144128:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768270.471439:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144256:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768270.471452:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768270.471456:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768270/real 1775768270] req@ffff90c203639180 x1862023742144256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768270.471463:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639180 x1862023742144256/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768270.471469:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768270.471470:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768270.471472:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144256:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775768271.495044:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768272.519096:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768273.543038:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768274.567010:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768275.591029:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768275.591041:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768275.591047:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768275.591060:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4164 00010000:00010000:3.0:1775768275.591074:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768275.591075:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768275.591084:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768275.591140:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497480 x1862023742144384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768275.591163:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768275.591166:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768275.591167:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768275.591170:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4164 00000100:00080000:1.0:1775768275.591173:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768275.591175:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768275.591184:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768275.591197:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494000 x1862023742144512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768275.591211:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742144384:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775768275.591225:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768275.591227:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768275.591229:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768275.591232:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4164 00000100:00080000:1.0:1775768275.591235:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768275.591238:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768275.591242:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768275.591252:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494380 x1862023742144640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768275.591258:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768275.591260:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768275.591261:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768275.591263:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4164 00000100:00080000:1.0:1775768275.591265:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768275.591268:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768275.591271:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768275.591277:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495f80 x1862023742144768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768275.591282:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768275.591321:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768275/real 1775768275] req@ffff90c203497480 x1862023742144384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768275.591336:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497480 x1862023742144384/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768275.591344:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768275.591347:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768275.591350:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742144384:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768275.591369:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144512:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768275.591381:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768275.591388:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768275/real 1775768275] req@ffff90c203494000 x1862023742144512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768275.591395:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494000 x1862023742144512/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768275.591400:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768275.591402:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768275.591404:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144512:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768275.591407:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144640:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768275.591416:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768275.591421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768275/real 1775768275] req@ffff90c203494380 x1862023742144640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768275.591428:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494380 x1862023742144640/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768275.591432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768275.591433:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768275.591435:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144640:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768275.591438:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144768:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768275.591447:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768275.591451:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768275/real 1775768275] req@ffff90c203495f80 x1862023742144768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768275.591457:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495f80 x1862023742144768/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768275.591464:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768275.591465:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768275.591467:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742144768:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768276.615088:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768277.639059:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768278.663076:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768279.687039:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768280.711007:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768280.711023:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768280.711031:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768280.711043:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4169 00000100:00080000:1.0:1775768280.711054:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768280.711062:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768280.711099:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496d80 x1862023742144896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1775768280.711114:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768280.711122:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768280.711125:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768280.711127:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768280.711130:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4169 00000100:00080000:1.0:1775768280.711133:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768280.711136:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768280.711166:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768280.711181:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497100 x1862023742145024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768280.711184:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742144896:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775768280.711192:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768280.711195:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768280.711196:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768280.711199:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4169 00000100:00080000:1.0:1775768280.711201:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768280.711203:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768280.711207:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768280.711220:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495880 x1862023742145152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768280.711225:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768280.711229:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768280.711230:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768280.711232:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4169 00000100:00080000:1.0:1775768280.711235:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768280.711237:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768280.711240:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768280.711245:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494e00 x1862023742145280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768280.711275:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768280.711321:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768280/real 1775768280] req@ffff90c203496d80 x1862023742144896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768280.711340:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496d80 x1862023742144896/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768280.711352:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768280.711357:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768280.711362:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742144896:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768280.711389:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145024:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768280.711407:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768280.711418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768280/real 1775768280] req@ffff90c203497100 x1862023742145024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768280.711430:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497100 x1862023742145024/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768280.711437:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768280.711441:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768280.711444:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145024:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768280.711449:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145152:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768280.711464:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768280.711472:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768280/real 1775768280] req@ffff90c203495880 x1862023742145152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768280.711482:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495880 x1862023742145152/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768280.711490:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768280.711493:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768280.711495:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145152:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768280.711502:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145280:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768280.711516:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768280.711524:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768280/real 1775768280] req@ffff90c203494e00 x1862023742145280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768280.711534:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494e00 x1862023742145280/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768280.711544:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768280.711547:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768280.711550:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145280:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768281.735042:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768282.759019:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768283.783000:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768284.807084:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768285.830974:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768285.830983:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768285.830987:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768285.830996:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4174 00000100:00080000:2.0:1775768285.831007:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768285.831013:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768285.831049:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be680 x1862023742145408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768285.831074:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768285.831078:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768285.831080:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768285.831084:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4174 00000100:00080000:2.0:1775768285.831087:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768285.831092:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768285.831109:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768285.831121:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bed80 x1862023742145536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768285.831130:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768285.831133:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768285.831135:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768285.831139:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4174 00000100:00080000:2.0:1775768285.831142:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768285.831146:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768285.831157:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768285.831169:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742145408:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768285.831169:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc700 x1862023742145664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768285.831187:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768285.831190:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768285.831192:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768285.831196:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4174 00000100:00080000:2.0:1775768285.831199:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768285.831202:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768285.831207:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768285.831218:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf100 x1862023742145792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768285.831245:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768285.831285:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768285/real 1775768285] req@ffff90c2034be680 x1862023742145408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768285.831305:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be680 x1862023742145408/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768285.831315:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768285.831320:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768285.831331:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742145408:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768285.831339:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145536:192.168.202.155@tcp:38:kworker.0 00010000:00010000:3.0:1775768285.831365:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1775768285.831370:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768285.831383:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768285/real 1775768285] req@ffff90c2034bed80 x1862023742145536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768285.831394:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bed80 x1862023742145536/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768285.831401:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768285.831404:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768285.831407:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bed80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145536:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768285.831439:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145664:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768285.831459:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768285.831470:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768285/real 1775768285] req@ffff90c2034bc700 x1862023742145664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768285.831481:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc700 x1862023742145664/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768285.831488:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768285.831492:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768285.831494:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145664:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768285.831500:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145792:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768285.831517:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768285.831526:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768285/real 1775768285] req@ffff90c2034bf100 x1862023742145792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768285.831536:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf100 x1862023742145792/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768285.831545:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768285.831548:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768285.831551:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742145792:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775768286.855067:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768287.879053:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768288.903093:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768289.929017:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768290.950979:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768290.950986:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768290.950989:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768290.950997:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4179 00000100:00080000:2.0:1775768290.951007:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768290.951012:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768290.951142:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bdf80 x1862023742145920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768290.951163:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768290.951165:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768290.951171:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768290.951173:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4179 00000100:00080000:2.0:1775768290.951176:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768290.951179:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768290.951187:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768290.951196:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd180 x1862023742146048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768290.951203:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1775768290.951205:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742145920:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775768290.951205:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768290.951208:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768290.951214:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4179 00000100:00080000:2.0:1775768290.951217:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768290.951218:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768290.951221:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768290.951229:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bc380 x1862023742146176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768290.951236:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768290.951238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768290.951239:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768290.951241:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4179 00000100:00080000:2.0:1775768290.951243:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768290.951245:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768290.951247:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768290.951253:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bce00 x1862023742146304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768290.951296:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768290.951332:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146176:192.168.202.155@tcp:38:kworker.0 00010000:00010000:1.0:1775768290.951353:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768290.951356:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146304:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768291.975100:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768292.999125:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768294.023063:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1775768294.023142:0:2413:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:3.0:1775768294.023180:0:2413:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1775768294.023196:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775768294.023236:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775768294.023245:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:3.0:1775768294.023253:0:2413:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775768294.023337:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768290/real 1775768294] req@ffff90c2034bd180 x1862023742146048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768294.023360:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd180 x1862023742146048/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768294.023371:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768294.023376:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768294.023380:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146048:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768294.023387:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768290/real 1775768294] req@ffff90c2034bc380 x1862023742146176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768294.023400:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bc380 x1862023742146176/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768294.023409:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768294.023412:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768294.023415:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bc380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146176:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768294.023421:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768290/real 1775768294] req@ffff90c2034bce00 x1862023742146304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768294.023431:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bce00 x1862023742146304/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768294.023440:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768294.023444:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768294.023447:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bce00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146304:192.168.202.155@tcp:8:kworker.0 00000100:00100000:0.0:1775768294.023452:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768290/real 1775768294] req@ffff90c2034bdf80 x1862023742145920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768294.023464:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bdf80 x1862023742145920/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768315 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768294.023472:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768294.023475:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768294.023478:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bdf80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742145920:192.168.202.155@tcp:250:kworker.0 00010000:00010000:3.0:1775768295.047041:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768296.070991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768296.071000:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768296.071013:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768296.071023:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4184 00000100:00080000:2.0:1775768296.071033:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768296.071039:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768296.071083:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bf480 x1862023742146432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768296.071109:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768296.071112:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768296.071114:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768296.071124:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4184 00000100:00080000:2.0:1775768296.071128:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768296.071132:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768296.071142:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768296.071155:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034be300 x1862023742146560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768296.071156:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742146432:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775768296.071171:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768296.071174:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768296.071176:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768296.071180:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4184 00000100:00080000:2.0:1775768296.071183:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768296.071187:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768296.071192:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768296.071202:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742146688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768296.071211:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768296.071216:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1775768296.071217:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775768296.071218:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768296.071223:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4184 00000100:00080000:2.0:1775768296.071227:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768296.071231:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768296.071237:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768296.071254:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768296/real 1775768296] req@ffff90c2034bf480 x1862023742146432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768296.071258:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742146816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768296.071284:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bf480 x1862023742146432/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768296.071295:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768296.071300:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768296.071304:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bf480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742146432:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768296.071324:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146560:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768296.071344:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768296.071356:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768296/real 1775768296] req@ffff90c2034be300 x1862023742146560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1775768296.071361:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768296.071376:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034be300 x1862023742146560/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768296.071385:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768296.071388:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768296.071392:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034be300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146560:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768296.071398:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146688:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768296.071414:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768296.071422:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768296/real 1775768296] req@ffff90c2034bca80 x1862023742146688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768296.071446:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742146688/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768296.071454:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768296.071457:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768296.071461:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146688:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768296.071466:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146816:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768296.071482:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768296.071490:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768296/real 1775768296] req@ffff90c2034bea00 x1862023742146816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768296.071500:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742146816/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768296.071514:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768296.071517:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768296.071519:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742146816:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768297.095076:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768298.119023:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768299.143007:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768300.167034:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768301.190978:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768301.190985:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768301.190989:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768301.190996:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4189 00000100:00080000:1.0:1775768301.191005:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768301.191011:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775768301.191016:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775768301.191044:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495c00 x1862023742146944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768301.191064:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768301.191066:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768301.191067:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768301.191069:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4189 00000100:00080000:1.0:1775768301.191071:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768301.191074:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768301.191080:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768301.191088:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203497b80 x1862023742147072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768301.191095:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768301.191097:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768301.191098:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768301.191101:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4189 00000100:00080000:1.0:1775768301.191104:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1775768301.191106:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742146944:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775768301.191108:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768301.191114:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768301.191119:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496300 x1862023742147200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768301.191126:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768301.191128:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768301.191129:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768301.191131:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4189 00000100:00080000:1.0:1775768301.191133:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768301.191135:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768301.191138:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768301.191144:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496a00 x1862023742147328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768301.191161:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768301.191190:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768301/real 1775768301] req@ffff90c203495c00 x1862023742146944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768301.191202:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495c00 x1862023742146944/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768301.191209:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768301.191211:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768301.191214:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742146944:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768301.191218:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147072:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768301.191230:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768301.191236:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768301/real 1775768301] req@ffff90c203497b80 x1862023742147072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768301.191242:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203497b80 x1862023742147072/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768301.191246:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768301.191248:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768301.191250:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203497b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147072:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768301.191265:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147200:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768301.191274:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768301.191279:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768301/real 1775768301] req@ffff90c203496300 x1862023742147200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768301.191285:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496300 x1862023742147200/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768301.191289:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768301.191291:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768301.191292:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147200:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768301.191295:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147328:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768301.191303:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768301.191308:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768301/real 1775768301] req@ffff90c203496a00 x1862023742147328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768301.191313:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496a00 x1862023742147328/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768301.191319:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768301.191320:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768301.191322:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147328:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768302.215017:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768303.239060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768304.263045:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768305.287076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1775768306.311026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768306.311038:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1775768306.311044:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768306.311055:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4194 00000100:00080000:2.0:1775768306.311069:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768306.311076:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768306.311126:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bd500 x1862023742147456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768306.311167:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768306.311171:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1775768306.311173:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768306.311178:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4194 00000100:00080000:2.0:1775768306.311182:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768306.311186:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768306.311198:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768306.311214:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bfb80 x1862023742147584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768306.311219:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742147456:192.168.202.155@tcp:250:kworker.0 00000100:00080000:2.0:1775768306.311234:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768306.311238:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1775768306.311240:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1775768306.311245:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4194 00000100:00080000:2.0:1775768306.311249:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768306.311253:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768306.311259:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768306.311273:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bea00 x1862023742147712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1775768306.311287:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1775768306.311290:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1775768306.311293:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1775768306.311294:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:2.0:1775768306.311299:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4194 00000100:00080000:2.0:1775768306.311302:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1775768306.311305:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:2.0:1775768306.311309:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:2.0:1775768306.311321:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c2034bca80 x1862023742147840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768306.311337:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768306/real 1775768306] req@ffff90c2034bd500 x1862023742147456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768306.311353:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bd500 x1862023742147456/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768306.311362:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768306.311366:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768306.311369:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bd500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742147456:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768306.311392:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147584:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768306.311408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00010000:00010000:1.0:1775768306.311418:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768306.311422:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768306/real 1775768306] req@ffff90c2034bfb80 x1862023742147584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768306.311436:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bfb80 x1862023742147584/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768306.311442:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768306.311444:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768306.311445:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bfb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147584:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768306.311448:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147712:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768306.311460:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768306.311467:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768306/real 1775768306] req@ffff90c2034bea00 x1862023742147712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768306.311488:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bea00 x1862023742147712/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768306.311501:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768306.311505:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768306.311507:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bea00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147712:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768306.311513:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147840:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768306.311533:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768306.311543:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768306/real 1775768306] req@ffff90c2034bca80 x1862023742147840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768306.311553:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c2034bca80 x1862023742147840/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768306.311562:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768306.311564:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768306.311566:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c2034bca80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742147840:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768307.335135:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768308.359076:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768309.383190:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768310.407069:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768311.430967:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768311.430974:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768311.430977:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768311.430984:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4199 00000100:00080000:1.0:1775768311.430994:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768311.431000:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775768311.431022:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1775768311.431031:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203496680 x1862023742147968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768311.431054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768311.431056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768311.431057:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768311.431060:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4199 00000100:00080000:1.0:1775768311.431062:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768311.431064:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768311.431071:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768311.431078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742148096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768311.431084:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768311.431087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768311.431088:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768311.431091:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4199 00000100:00080000:1.0:1775768311.431092:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768311.431094:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768311.431098:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768311.431102:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742147968:192.168.202.155@tcp:250:kworker.0 00000100:00100000:1.0:1775768311.431106:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742148224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768311.431113:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768311.431115:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768311.431115:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768311.431118:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4199 00000100:00080000:1.0:1775768311.431119:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768311.431121:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768311.431124:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768311.431130:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742148352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768311.431156:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768311.431187:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768311/real 1775768311] req@ffff90c203496680 x1862023742147968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768311.431204:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203496680 x1862023742147968/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768311.431214:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768311.431218:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768311.431222:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203496680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742147968:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768311.431230:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148096:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768311.431248:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768311.431259:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768311/real 1775768311] req@ffff90c203494a80 x1862023742148096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768311.431270:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742148096/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768311.431278:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768311.431281:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768311.431283:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148096:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768311.431301:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148224:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768311.431314:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768311.431320:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768311/real 1775768311] req@ffff90c203494700 x1862023742148224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768311.431328:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742148224/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768311.431335:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768311.431338:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768311.431341:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148224:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768311.431346:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148352:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768311.431362:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768311.431371:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768311/real 1775768311] req@ffff90c203495500 x1862023742148352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768311.431383:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742148352/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768311.431389:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768311.431391:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768311.431393:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148352:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775768312.455016:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768313.479030:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768314.503018:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768315.527041:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768316.550975:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768316.550985:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1775768316.550990:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768316.551001:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4205 00000100:00080000:1.0:1775768316.551012:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00010000:00010000:3.0:1775768316.551019:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1775768316.551020:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:1.0:1775768316.551058:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495180 x1862023742148480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768316.551083:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768316.551087:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1775768316.551088:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768316.551092:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4205 00000100:00080000:1.0:1775768316.551096:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768316.551099:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768316.551107:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768316.551119:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203495500 x1862023742148608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768316.551125:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768316.551127:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1775768316.551130:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1775768316.551132:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742148480:192.168.202.155@tcp:250:kworker.0 00000100:00080000:1.0:1775768316.551134:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4205 00000100:00080000:1.0:1775768316.551139:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768316.551142:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768316.551145:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1775768316.551153:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494700 x1862023742148736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1775768316.551158:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1775768316.551160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1775768316.551161:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1775768316.551164:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4205 00000100:00080000:1.0:1775768316.551176:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1775768316.551178:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:1.0:1775768316.551181:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:1.0:1775768316.551186:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203494a80 x1862023742148864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1775768316.551191:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768316.551223:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768316/real 1775768316] req@ffff90c203495180 x1862023742148480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768316.551235:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495180 x1862023742148480/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768316.551241:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768316.551244:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768316.551247:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742148480:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768316.551261:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148608:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768316.551274:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768316.551297:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768316/real 1775768316] req@ffff90c203495500 x1862023742148608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768316.551308:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203495500 x1862023742148608/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768316.551316:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768316.551323:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768316.551326:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203495500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148608:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768316.551332:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148736:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768316.551349:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768316.551358:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768316/real 1775768316] req@ffff90c203494700 x1862023742148736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768316.551369:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494700 x1862023742148736/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768316.551376:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768316.551379:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768316.551382:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148736:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768316.551387:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148864:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768316.551403:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768316.551412:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768316/real 1775768316] req@ffff90c203494a80 x1862023742148864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768316.551423:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203494a80 x1862023742148864/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768316.551432:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768316.551434:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768316.551438:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203494a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742148864:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768317.575052:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768318.599017:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768319.623017:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768320.647042:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768321.670986:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768321.670993:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768321.670997:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768321.671004:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4210 00000100:00080000:0.0:1775768321.671013:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768321.671019:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768321.671021:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768321.671049:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638000 x1862023742148992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768321.671068:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768321.671071:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768321.671072:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768321.671074:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4210 00000100:00080000:0.0:1775768321.671077:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768321.671079:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768321.671086:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768321.671094:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639500 x1862023742149120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768321.671100:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742148992:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775768321.671102:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768321.671107:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768321.671109:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768321.671111:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4210 00000100:00080000:0.0:1775768321.671113:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768321.671115:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768321.671119:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768321.671125:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203638380 x1862023742149248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768321.671131:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768321.671133:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768321.671134:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768321.671136:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4210 00000100:00080000:0.0:1775768321.671138:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768321.671139:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768321.671143:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768321.671149:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363a680 x1862023742149376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768321.671153:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768321.671195:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768321/real 1775768321] req@ffff90c203638000 x1862023742148992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768321.671212:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638000 x1862023742148992/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768321.671222:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768321.671226:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768321.671229:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742148992:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768321.671249:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149120:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768321.671266:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768321.671274:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768321/real 1775768321] req@ffff90c203639500 x1862023742149120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768321.671283:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639500 x1862023742149120/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768321.671288:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768321.671290:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768321.671291:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149120:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768321.671295:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149248:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768321.671304:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768321.671309:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768321/real 1775768321] req@ffff90c203638380 x1862023742149248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768321.671315:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203638380 x1862023742149248/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768321.671326:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768321.671328:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768321.671329:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203638380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149248:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768321.671332:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149376:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768321.671342:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768321.671347:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768321/real 1775768321] req@ffff90c20363a680 x1862023742149376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768321.671353:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363a680 x1862023742149376/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768321.671363:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768321.671365:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768321.671366:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149376:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768322.694999:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768323.719023:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768324.743077:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768325.767053:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768326.790973:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768326.790980:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768326.790983:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768326.790991:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4215 00000100:00080000:0.0:1775768326.791000:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768326.791005:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:3.0:1775768326.791023:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768326.791036:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b100 x1862023742149504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768326.791054:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768326.791056:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768326.791058:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768326.791060:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4215 00000100:00080000:0.0:1775768326.791062:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768326.791064:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768326.791071:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768326.791078:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742149632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768326.791086:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1775768326.791089:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742149504:192.168.202.155@tcp:250:kworker.0 00000100:00080000:0.0:1775768326.791090:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768326.791094:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768326.791100:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4215 00000100:00080000:0.0:1775768326.791103:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768326.791107:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768326.791111:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768326.791122:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742149760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768326.791131:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768326.791134:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768326.791136:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768326.791140:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4215 00000100:00080000:0.0:1775768326.791144:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1775768326.791148:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00080000:0.0:1775768326.791149:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768326.791155:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768326.791167:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742149888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768326.791185:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768326/real 1775768326] req@ffff90c20363b100 x1862023742149504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768326.791202:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b100 x1862023742149504/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768326.791210:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768326.791214:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768326.791217:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742149504:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768326.791236:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149632:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768326.791254:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768326.791264:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768326/real 1775768326] req@ffff90c20363b480 x1862023742149632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768326.791275:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742149632/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768326.791283:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768326.791285:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768326.791288:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149632:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768326.791294:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149760:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768326.791308:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768326.791316:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768326/real 1775768326] req@ffff90c203639f80 x1862023742149760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768326.791325:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742149760/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768326.791332:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768326.791334:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768326.791337:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149760:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768326.791342:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149888:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768326.791356:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768326.791364:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768326/real 1775768326] req@ffff90c20363bb80 x1862023742149888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768326.791373:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742149888/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768326.791381:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768326.791385:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768326.791387:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742149888:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768327.815107:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768328.839106:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768329.863033:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768330.887079:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768331.910991:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768331.910999:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1775768331.911004:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768331.911014:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4220 00000100:00080000:0.0:1775768331.911024:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768331.911031:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:0.0:1775768331.911071:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b800 x1862023742150016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:1.0:1775768331.911072:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1775768331.911107:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768331.911111:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1775768331.911113:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768331.911117:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4220 00000100:00080000:0.0:1775768331.911121:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768331.911124:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768331.911133:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1775768331.911147:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363bb80 x1862023742150144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768331.911156:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768331.911160:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1775768331.911162:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768331.911166:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4220 00000100:00080000:0.0:1775768331.911170:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768331.911174:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768331.911179:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1775768331.911186:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742150016:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768331.911191:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c203639f80 x1862023742150272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1775768331.911202:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1775768331.911205:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1775768331.911207:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1775768331.911211:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4220 00000100:00080000:0.0:1775768331.911214:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1775768331.911218:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:0.0:1775768331.911223:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:0.0:1775768331.911234:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c20363b480 x1862023742150400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1775768331.911250:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768331.911288:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768331/real 1775768331] req@ffff90c20363b800 x1862023742150016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768331.911307:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b800 x1862023742150016/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768331.911316:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768331.911320:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768331.911324:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742150016:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768331.911331:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150144:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768331.911351:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768331.911362:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768331/real 1775768331] req@ffff90c20363bb80 x1862023742150144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768331.911374:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363bb80 x1862023742150144/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768331.911382:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768331.911385:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768331.911389:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150144:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768331.911411:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150272:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768331.911430:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768331.911440:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768331/real 1775768331] req@ffff90c203639f80 x1862023742150272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768331.911451:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c203639f80 x1862023742150272/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768331.911464:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768331.911466:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768331.911469:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c203639f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150272:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768331.911475:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150400:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768331.911493:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768331.911501:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768331/real 1775768331] req@ffff90c20363b480 x1862023742150400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768331.911511:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c20363b480 x1862023742150400/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768331.911518:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768331.911520:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768331.911522:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c20363b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150400:192.168.202.155@tcp:8:kworker.0 00010000:00010000:3.0:1775768332.935058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768333.959060:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768334.983061:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768336.007083:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768337.030987:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768337.030997:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768337.031002:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768337.031011:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4225 00000100:00080000:3.0:1775768337.031023:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768337.031030:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00010000:00010000:1.0:1775768337.031031:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:3.0:1775768337.031068:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742150528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768337.031095:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768337.031099:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768337.031100:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768337.031104:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4225 00000100:00080000:3.0:1775768337.031108:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768337.031112:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768337.031120:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768337.031133:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742150656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768337.031143:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768337.031146:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768337.031147:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768337.031151:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4225 00000100:00080000:3.0:1775768337.031154:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768337.031157:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:2.0:1775768337.031161:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742150528:192.168.202.155@tcp:250:kworker.0 00000008:00100000:3.0:1775768337.031161:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768337.031180:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742150784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768337.031189:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768337.031192:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768337.031193:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768337.031196:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4225 00000100:00080000:3.0:1775768337.031206:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768337.031209:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768337.031213:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000800:00000100:2.0:1775768337.031216:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768337.031222:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742150912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768337.031248:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768337/real 1775768337] req@ffff90c218da3480 x1862023742150528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768337.031264:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742150528/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768337.031271:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768337.031274:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768337.031276:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742150528:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768337.031282:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150656:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768337.031297:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768337.031304:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768337/real 1775768337] req@ffff90c218da1f80 x1862023742150656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768337.031311:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742150656/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768337.031316:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768337.031318:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768337.031320:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150656:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768337.031336:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150784:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768337.031345:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768337.031350:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768337/real 1775768337] req@ffff90c218da0000 x1862023742150784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768337.031356:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742150784/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768337.031361:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768337.031363:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768337.031364:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150784:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768337.031369:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150912:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768337.031384:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768337.031392:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768337/real 1775768337] req@ffff90c218da3100 x1862023742150912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768337.031403:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742150912/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768337.031412:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768337.031415:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768337.031418:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742150912:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768338.054991:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768339.079053:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768340.102979:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768341.127034:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768342.151020:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768342.151029:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768342.151034:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768342.151044:0:37:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4230 00000100:00080000:3.0:1775768342.151054:0:37:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768342.151061:0:37:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768342.151095:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742151040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768342.151115:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768342.151118:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768342.151120:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768342.151124:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4230 00000100:00080000:3.0:1775768342.151128:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775768342.151131:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742151040:192.168.202.155@tcp:250:kworker.0 00000100:00080000:3.0:1775768342.151134:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768342.151146:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768342.151159:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742151168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768342.151170:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768342.151173:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768342.151175:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768342.151178:0:37:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4230 00000100:00080000:3.0:1775768342.151183:0:37:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768342.151187:0:37:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768342.151192:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1775768342.151193:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:3.0:1775768342.151205:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742151296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768342.151215:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768342.151218:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768342.151220:0:37:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768342.151225:0:37:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4230 00000100:00080000:3.0:1775768342.151228:0:37:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1775768342.151233:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768342/real 1775768342] req@ffff90c218da0a80 x1862023742151040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1775768342.151234:0:37:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768342.151241:0:37:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768342.151256:0:37:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742151424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768342.151260:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742151040/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768342.151283:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768342.151288:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768342.151293:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742151040:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768342.151319:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151168:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768342.151340:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768342.151352:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768342/real 1775768342] req@ffff90c218da0380 x1862023742151168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768342.151363:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742151168/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768342.151377:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768342.151380:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768342.151384:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151168:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768342.151389:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151296:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768342.151408:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768342.151418:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768342/real 1775768342] req@ffff90c218da1880 x1862023742151296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768342.151428:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742151296/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768342.151435:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768342.151438:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768342.151441:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151296:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768342.151446:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151424:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768342.151462:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768342.151475:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768342/real 1775768342] req@ffff90c218da0700 x1862023742151424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768342.151486:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742151424/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768342.151495:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768342.151498:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768342.151501:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151424:192.168.202.155@tcp:8:kworker.0 00010000:00010000:0.0:1775768342.151548:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768343.175111:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768344.199115:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768345.223244:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768346.247140:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768347.271002:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768347.271011:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768347.271014:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768347.271024:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4235 00000100:00080000:3.0:1775768347.271034:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768347.271040:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768347.271073:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3800 x1862023742151552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768347.271100:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768347.271103:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768347.271105:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768347.271108:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4235 00000100:00080000:3.0:1775768347.271111:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768347.271114:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768347.271123:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768347.271133:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da2d80 x1862023742151680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768347.271141:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768347.271144:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768347.271146:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768347.271149:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4235 00000100:00080000:3.0:1775768347.271152:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768347.271155:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768347.271159:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768347.271167:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0e00 x1862023742151808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768347.271174:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768347.271177:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768347.271179:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768347.271182:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4235 00000100:00080000:3.0:1775768347.271185:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768347.271200:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768347.271205:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768347.271214:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0700 x1862023742151936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1775768347.272570:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742151552:192.168.202.155@tcp:250:kworker.0 00000800:00000100:2.0:1775768347.272626:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768347.272659:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768347/real 1775768347] req@ffff90c218da3800 x1862023742151552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768347.272676:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3800 x1862023742151552/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768347.272684:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768347.272688:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768347.272692:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742151552:192.168.202.155@tcp:250:kworker.0 00000100:00100000:2.0:1775768347.272698:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151680:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768347.272715:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768347.272724:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768347/real 1775768347] req@ffff90c218da2d80 x1862023742151680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768347.272734:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da2d80 x1862023742151680/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768347.272741:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768347.272744:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768347.272747:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da2d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151680:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768347.272751:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151808:192.168.202.155@tcp:38:kworker.0 00000800:00000100:2.0:1775768347.272766:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768347.272774:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768347/real 1775768347] req@ffff90c218da0e00 x1862023742151808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768347.272784:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0e00 x1862023742151808/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768347.272791:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768347.272793:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768347.272796:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151808:192.168.202.155@tcp:38:kworker.0 00000100:00100000:2.0:1775768347.272801:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151936:192.168.202.155@tcp:8:kworker.0 00000800:00000100:2.0:1775768347.272816:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:2.0:1775768347.272823:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768347/real 1775768347] req@ffff90c218da0700 x1862023742151936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1775768347.272833:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0700 x1862023742151936/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1775768347.272842:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1775768347.272845:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:2.0:1775768347.272847:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742151936:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768347.274371:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768348.295058:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768349.319067:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768350.345284:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768351.370854:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768352.390976:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768352.390984:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768352.390989:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768352.390998:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4240 00000100:00080000:3.0:1775768352.391008:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768352.391014:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768352.391047:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1880 x1862023742152064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768352.391081:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768352.391085:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768352.391086:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768352.391090:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4240 00000100:00080000:3.0:1775768352.391093:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768352.391096:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768352.391105:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768352.391114:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0380 x1862023742152192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768352.391124:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768352.391127:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768352.391129:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768352.391132:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4240 00000100:00080000:3.0:1775768352.391134:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768352.391138:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768352.391141:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768352.391148:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0a80 x1862023742152320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768352.391157:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768352.391159:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768352.391161:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768352.391163:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4240 00000100:00080000:3.0:1775768352.391166:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768352.391168:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768352.391172:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768352.391181:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742152448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1775768352.391322:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742152064:192.168.202.155@tcp:250:kworker.0 00000800:00000100:0.0:1775768352.391373:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768352.391406:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768352/real 1775768352] req@ffff90c218da1880 x1862023742152064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768352.391424:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1880 x1862023742152064/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768352.391434:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768352.391438:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768352.391442:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742152064:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768352.391450:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152192:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768352.391480:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768352.391491:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768352/real 1775768352] req@ffff90c218da0380 x1862023742152192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768352.391514:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0380 x1862023742152192/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768352.391521:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768352.391524:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768352.391527:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152192:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768352.391533:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152320:192.168.202.155@tcp:38:kworker.0 00000800:00000100:0.0:1775768352.391548:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768352.391556:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768352/real 1775768352] req@ffff90c218da0a80 x1862023742152320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768352.391567:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0a80 x1862023742152320/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768352.391574:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768352.391577:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768352.391580:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152320:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768352.391586:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152448:192.168.202.155@tcp:8:kworker.0 00000800:00000100:0.0:1775768352.391612:0:2424:0:(socklnd_cb.c:976:ksocknal_launch_packet()) No usable routes to 12345-192.168.202.155@tcp 00000100:00100000:0.0:1775768352.391621:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768352/real 1775768352] req@ffff90c218da3100 x1862023742152448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768352.391633:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742152448/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768352.391642:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768352.391644:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768352.391647:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152448:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768352.392132:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768353.416596:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1775768354.439028:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768355.464820:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768356.487106:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1775768357.511866:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8b4cb7d5-7b49-44b3-b066-941aae62e872->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768357.511912:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1775768357.513235:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768357.513254:0:14408:0:(import.c:509:import_select_connection()) MGC192.168.202.155@tcp: connect to NID 192.168.202.155@tcp last attempt 4245 00000100:00080000:3.0:1775768357.513270:0:14408:0:(import.c:588:import_select_connection()) MGC192.168.202.155@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768357.513278:0:14408:0:(import.c:623:import_select_connection()) MGC192.168.202.155@tcp: import ffff90c210b78800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000100:00100000:3.0:1775768357.513330:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da0000 x1862023742152576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768357.513358:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768357.513362:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1775768357.513364:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768357.513369:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4245 00000100:00080000:3.0:1775768357.513372:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768357.513376:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0000-mdc-ffff90c203172800: import ffff90c219571800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768357.513389:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768357.513402:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da1f80 x1862023742152704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768357.513444:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768357.513448:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1775768357.513449:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768357.513453:0:14408:0:(import.c:509:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4245 00000100:00080000:3.0:1775768357.513457:0:14408:0:(import.c:588:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768357.513461:0:14408:0:(import.c:623:import_select_connection()) lustre-MDT0001-mdc-ffff90c203172800: import ffff90c21959b000 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768357.513467:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xae7e5e7be374d3b8 ocd_version: 34681676 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1775768357.513478:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3480 x1862023742152832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1775768357.513487:0:14408:0:(pinger.c:229:ptlrpc_pinger_process_import()) 8d5ce1d6-a53c-4efe-b6ac-3d35d5106778->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1775768357.513491:0:14408:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1775768357.513497:0:14408:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1775768357.513502:0:14408:0:(import.c:509:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: connect to NID 192.168.202.155@tcp last attempt 4245 00000100:00080000:3.0:1775768357.513506:0:14408:0:(import.c:588:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1775768357.513510:0:14408:0:(import.c:623:import_select_connection()) lustre-OST0000-osc-ffff90c203172800: import ffff90c2034c7800 using connection 192.168.202.155@tcp/192.168.202.155@tcp 00000008:00100000:3.0:1775768357.513515:0:14408:0:(osc_request.c:3829:osc_reconnect()) ocd_connect_flags: 0xa0465af2e3640038 ocd_version: 34681676 ocd_grant: 54771712, lost: 0. 00000100:00100000:3.0:1775768357.514021:0:14408:0:(import.c:825:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff90c218da3100 x1862023742152960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:3.0:1775768357.514679:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1775768357.515373:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742152576:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768357.515530:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768357.515574:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768357.515606:0:2424:0:(client.c:1904:ptlrpc_send_new_req()) Sending RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152960:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768358.538470:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1775768359.559024:0:10909:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1775768360.583185:0:10910:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1775768360.583283:0:2412:0:(lib-socket.c:454:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.202.155:988 00000400:00000100:1.0:1775768360.583324:0:2412:0:(acceptor.c:111:lnet_connect_console_error()) Connection to 192.168.202.155@tcp at host 192.168.202.155 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1775768360.583352:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775768360.583402:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775768360.583412:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000800:00000100:1.0:1775768360.583419:0:2412:0:(socklnd_cb.c:422:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.202.155@tcp 00000100:00100000:0.0:1775768360.584752:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768357/real 1775768360] req@ffff90c218da0000 x1862023742152576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768360.584785:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da0000 x1862023742152576/t0(0) o250->MGC192.168.202.155@tcp@192.168.202.155@tcp:26/25 lens 520/544 e 0 to 1 dl 1775768382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768360.584795:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c210b78800 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768360.584800:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of MGS on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768360.584805:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da0000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8b4cb7d5-7b49-44b3-b066-941aae62e872:2424:1862023742152576:192.168.202.155@tcp:250:kworker.0 00000100:00100000:0.0:1775768360.584814:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768357/real 1775768360] req@ffff90c218da1f80 x1862023742152704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768360.584825:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da1f80 x1862023742152704/t0(0) o38->lustre-MDT0000-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768360.584834:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c219571800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768360.584838:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768360.584841:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da1f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152704:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768360.584856:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768357/real 1775768360] req@ffff90c218da3480 x1862023742152832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768360.584869:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3480 x1862023742152832/t0(0) o38->lustre-MDT0001-mdc-ffff90c203172800@192.168.202.155@tcp:12/10 lens 520/544 e 0 to 1 dl 1775768382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768360.584904:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c21959b000 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768360.584912:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768360.584915:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152832:192.168.202.155@tcp:38:kworker.0 00000100:00100000:0.0:1775768360.584922:0:2424:0:(client.c:2479:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1775768357/real 1775768360] req@ffff90c218da3100 x1862023742152960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1775768360.584934:0:2424:0:(client.c:2510:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff90c218da3100 x1862023742152960/t0(0) o8->lustre-OST0000-osc-ffff90c203172800@192.168.202.155@tcp:28/4 lens 520/544 e 0 to 1 dl 1775768382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1775768360.584942:0:2424:0:(import.c:65:import_set_state_nolock()) ffff90c2034c7800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1775768360.584945:0:2424:0:(import.c:1506:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.202.155@tcp failed (-110) 00000100:00100000:0.0:1775768360.584949:0:2424:0:(client.c:2390:ptlrpc_check_set()) Completed RPC req@ffff90c218da3100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:8d5ce1d6-a53c-4efe-b6ac-3d35d5106778:2424:1862023742152960:192.168.202.155@tcp:8:kworker.0 00010000:00010000:1.0:1775768361.609269:0:15251:0:(ldlm_lockd.c:2797:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 78247 lines, 78247 kept, 0 dropped, 0 bad.